Now forwards create instance requests to child zones.

Refactored nova.compute.api.create() to support deferred db entry creation.

NOTE: created instances using the ZoneAwareScheduler result in two db entries: one in the parent zone, one in the child zone. This will be fixed in the next branch which will deal with reservation ID's and a new POST /zone/server OS API command. All the other schedulers work as advertised.

10 files changed, 447 insertions, 102 deletions

diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py
index d8fb5265b..c116e4220 100644
--- a/nova/api/openstack/__init__.py
+++ b/nova/api/openstack/__init__.py
@@ -101,7 +101,7 @@ class APIRouter(base_wsgi.Router):
             mapper.resource("zone", "zones",
                         controller=zones.create_resource(),
                         collection={'detail': 'GET', 'info': 'GET',
-                                    'select': 'GET'})
+                                    'select': 'POST'})
 
             mapper.resource("user", "users",
                         controller=users.create_resource(),
diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py
index 82d8be4aa..9cf5e8721 100644
--- a/nova/api/openstack/servers.py
+++ b/nova/api/openstack/servers.py
@@ -153,6 +153,7 @@ class Controller(object):
             msg = _("Server name is not defined")
             return exc.HTTPBadRequest(msg)
 
+        zone_blob = body['server'].get('blob')
         name = body['server']['name']
         self._validate_server_name(name)
         name = name.strip()
@@ -172,7 +173,8 @@ class Controller(object):
                 key_data=key_data,
                 metadata=body['server'].get('metadata', {}),
                 injected_files=injected_files,
-                admin_password=password)
+                admin_password=password,
+                zone_blob=zone_blob)
         except quota.QuotaError as error:
             self._handle_quota_error(error)
         except exception.ImageNotFound as error:
diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py
index 8061b3b67..b2f7898cb 100644
--- a/nova/api/openstack/zones.py
+++ b/nova/api/openstack/zones.py
@@ -27,9 +27,6 @@ from nova.scheduler import api
 
 
 FLAGS = flags.FLAGS
-flags.DEFINE_string('build_plan_encryption_key',
-        None,
-        '128bit (hex) encryption key for scheduler build plans.')
 
 
 LOG = logging.getLogger('nova.api.openstack.zones')
@@ -53,6 +50,14 @@ def _scrub_zone(zone):
                     'deleted', 'deleted_at', 'updated_at'))
 
 
+def check_encryption_key(func):
+    def wrapped(*args, **kwargs):
+        if not FLAGS.build_plan_encryption_key:
+            raise exception.Error(_("--build_plan_encryption_key not set"))
+        return func(*args, **kwargs)
+    return wrapped
+
+
 class Controller(object):
 
     def index(self, req):
@@ -103,19 +108,13 @@ class Controller(object):
         zone = api.zone_update(context, zone_id, body["zone"])
         return dict(zone=_scrub_zone(zone))
 
-    def select(self, req):
+    @check_encryption_key
+    def select(self, req, body):
         """Returns a weighted list of costs to create instances
            of desired capabilities."""
         ctx = req.environ['nova.context']
-        qs = req.environ['QUERY_STRING']
-        param_dict = urlparse.parse_qs(qs)
-        param_dict.pop("fresh", None)
-        # parse_qs returns a dict where the values are lists,
-        # since query strings can have multiple values for the
-        # same key. We need to convert that to single values.
-        for key in param_dict:
-            param_dict[key] = param_dict[key][0]
-        build_plan = api.select(ctx, specs=param_dict)
+        specs = json.loads(body)
+        build_plan = api.select(ctx, specs=specs)
         cooked = self._scrub_build_plan(build_plan)
         return {"weights": cooked}
 
@@ -123,9 +122,6 @@ class Controller(object):
         """Remove all the confidential data and return a sanitized
         version of the build plan. Include an encrypted full version
         of the weighting entry so we can get back to it later."""
-        if not FLAGS.build_plan_encryption_key:
-            raise exception.FlagNotSet(flag='build_plan_encryption_key')
-
         encryptor = crypto.encryptor(FLAGS.build_plan_encryption_key)
         cooked = []
         for entry in build_plan:
diff --git a/nova/compute/api.py b/nova/compute/api.py
index 4f327fab1..b0949a729 100644
--- a/nova/compute/api.py
+++ b/nova/compute/api.py
@@ -128,18 +128,16 @@ class API(base.Base):
                 LOG.warn(msg)
                 raise quota.QuotaError(msg, "MetadataLimitExceeded")
 
-    def create(self, context, instance_type,
+    def _check_create_parameters(self, context, instance_type,
                image_href, kernel_id=None, ramdisk_id=None,
                min_count=1, max_count=1,
                display_name='', display_description='',
                key_name=None, key_data=None, security_group='default',
                availability_zone=None, user_data=None, metadata={},
-               injected_files=None,
-               admin_password=None):
-        """Create the number and type of instances requested.
+               injected_files=None, admin_password=None, zone_blob=None):
+        """Verify all the input parameters regardless of the provisioning
+        strategy being performed."""
 
-        Verifies that quota and other arguments are valid.
-        """
         if not instance_type:
             instance_type = instance_types.get_default_instance_type()
 
@@ -225,63 +223,145 @@ class API(base.Base):
             'metadata': metadata,
             'availability_zone': availability_zone,
             'os_type': os_type}
-        elevated = context.elevated()
-        instances = []
-        LOG.debug(_("Going to run %s instances..."), num_instances)
-        for num in range(num_instances):
-            instance = dict(mac_address=utils.generate_mac(),
-                            launch_index=num,
-                            **base_options)
-            instance = self.db.instance_create(context, instance)
-            instance_id = instance['id']
 
-            elevated = context.elevated()
-            if not security_groups:
-                security_groups = []
-            for security_group_id in security_groups:
-                self.db.instance_add_security_group(elevated,
-                                                    instance_id,
-                                                    security_group_id)
-
-            # Set sane defaults if not specified
-            updates = dict(hostname=self.hostname_factory(instance_id))
-            if (not hasattr(instance, 'display_name') or
-                    instance.display_name is None):
-                updates['display_name'] = "Server %s" % instance_id
-
-            instance = self.update(context, instance_id, **updates)
-            instances.append(instance)
+        return (num_instances, base_options, security_groups)
+
+    def create_db_entry_for_new_instance(self, context, base_options,
+             security_groups, num=1):
+        """Create an entry in the DB for this new instance,
+        including any related table updates (such as security
+        groups, MAC address, etc). This will called by create()
+        in the majority of situations, but all-at-once style
+        Schedulers may initiate the call."""
+        instance = dict(mac_address=utils.generate_mac(),
+                        launch_index=num,
+                        **base_options)
+        instance = self.db.instance_create(context, instance)
+        instance_id = instance['id']
 
-            pid = context.project_id
-            uid = context.user_id
-            LOG.debug(_("Casting to scheduler for %(pid)s/%(uid)s's"
-                    " instance %(instance_id)s") % locals())
+        elevated = context.elevated()
+        if not security_groups:
+            security_groups = []
+        for security_group_id in security_groups:
+            self.db.instance_add_security_group(elevated,
+                                                instance_id,
+                                                security_group_id)
 
-            # NOTE(sandy): For now we're just going to pass in the
-            # instance_type record to the scheduler. In a later phase
-            # we'll be ripping this whole for-loop out and deferring the
-            # creation of the Instance record. At that point all this will
-            # change.
-            rpc.cast(context,
-                     FLAGS.scheduler_topic,
-                     {"method": "run_instance",
-                      "args": {"topic": FLAGS.compute_topic,
-                               "instance_id": instance_id,
-                               "request_spec": {
-                                        'instance_type': instance_type,
-                                        'filter':
-                                            'nova.scheduler.host_filter.'
-                                            'InstanceTypeFilter',
-                               },
-                               "availability_zone": availability_zone,
-                               "injected_files": injected_files,
-                               "admin_password": admin_password,
-                              },
-                     })
+        # Set sane defaults if not specified
+        updates = dict(hostname=self.hostname_factory(instance_id))
+        if (not hasattr(instance, 'display_name') or
+                instance.display_name is None):
+            updates['display_name'] = "Server %s" % instance_id
+
+        instance = self.update(context, instance_id, **updates)
 
         for group_id in security_groups:
             self.trigger_security_group_members_refresh(elevated, group_id)
 
+        return instance
+
+    def _ask_scheduler_to_create_instance(self, context, base_options,
+                                          instance_type, zone_blob,
+                                          availability_zone, injected_files,
+                                          admin_password,
+                                          instance_id=None, num_instances=1):
+        """Send the run_instance request to the schedulers for processing."""
+        pid = context.project_id
+        uid = context.user_id
+        if instance_id:
+            LOG.debug(_("Casting to scheduler for %(pid)s/%(uid)s's"
+                    " instance %(instance_id)s (single-shot)") % locals())
+        else:
+            LOG.debug(_("Casting to scheduler for %(pid)s/%(uid)s's"
+                    " (all-at-once)") % locals())
+
+        filter_class = 'nova.scheduler.host_filter.InstanceTypeFilter'
+        request_spec = {
+            'instance_properties': base_options,
+            'instance_type': instance_type,
+            'filter': filter_class,
+            'blob': zone_blob,
+            'num_instances': num_instances
+        }
+
+        rpc.cast(context,
+                 FLAGS.scheduler_topic,
+                 {"method": "run_instance",
+                  "args": {"topic": FLAGS.compute_topic,
+                           "instance_id": instance_id,
+                           "request_spec": request_spec,
+                           "availability_zone": availability_zone,
+                           "admin_password": admin_password,
+                           "injected_files": injected_files}})
+
+    def create_all_at_once(self, context, instance_type,
+               image_href, kernel_id=None, ramdisk_id=None,
+               min_count=1, max_count=1,
+               display_name='', display_description='',
+               key_name=None, key_data=None, security_group='default',
+               availability_zone=None, user_data=None, metadata={},
+               injected_files=None, admin_password=None, zone_blob=None):
+        """Provision the instances by passing the whole request to
+        the Scheduler for execution. Returns a Reservation ID
+        related to the creation of all of these instances."""
+        num_instances, base_options, security_groups = \
+                    self._check_create_parameters(
+                               context, instance_type,
+                               image_href, kernel_id, ramdisk_id,
+                               min_count, max_count,
+                               display_name, display_description,
+                               key_name, key_data, security_group,
+                               availability_zone, user_data, metadata,
+                               injected_files, admin_password, zone_blob)
+
+        self._ask_scheduler_to_create_instance(context, base_options,
+                                      instance_type, zone_blob,
+                                      availability_zone, injected_files,
+                                      admin_password,
+                                      num_instances=num_instances)
+
+        return base_options['reservation_id']
+
+    def create(self, context, instance_type,
+               image_href, kernel_id=None, ramdisk_id=None,
+               min_count=1, max_count=1,
+               display_name='', display_description='',
+               key_name=None, key_data=None, security_group='default',
+               availability_zone=None, user_data=None, metadata={},
+               injected_files=None, admin_password=None, zone_blob=None):
+        """
+        Provision the instances by sending off a series of single
+        instance requests to the Schedulers. This is fine for trival
+        Scheduler drivers, but may remove the effectiveness of the
+        more complicated drivers.
+
+        Returns a list of instance dicts.
+        """
+
+        num_instances, base_options, security_groups = \
+                    self._check_create_parameters(
+                               context, instance_type,
+                               image_href, kernel_id, ramdisk_id,
+                               min_count, max_count,
+                               display_name, display_description,
+                               key_name, key_data, security_group,
+                               availability_zone, user_data, metadata,
+                               injected_files, admin_password, zone_blob)
+
+        instances = []
+        LOG.debug(_("Going to run %s instances..."), num_instances)
+        for num in range(num_instances):
+            instance = self.create_db_entry_for_new_instance(context,
+                                    base_options, security_groups, num=num)
+            instances.append(instance)
+            instance_id = instance['id']
+
+            self._ask_scheduler_to_create_instance(context, base_options,
+                                          instance_type, zone_blob,
+                                          availability_zone, injected_files,
+                                          admin_password,
+                                          instance_id=instance_id)
+
         return [dict(x.iteritems()) for x in instances]
 
     def has_finished_migration(self, context, instance_id):
diff --git a/nova/flags.py b/nova/flags.py
index d5090edba..a8f16c6bb 100644
--- a/nova/flags.py
+++ b/nova/flags.py
@@ -381,3 +381,5 @@ DEFINE_string('zone_name', 'nova', 'name of this zone')
 DEFINE_list('zone_capabilities',
                 ['hypervisor=xenserver;kvm', 'os=linux;windows'],
                  'Key/Multi-value list representng capabilities of this zone')
+DEFINE_string('build_plan_encryption_key', None,
+        '128bit (hex) encryption key for scheduler build plans.')
diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py
index 55f8e0a6d..09e7c9140 100644
--- a/nova/scheduler/api.py
+++ b/nova/scheduler/api.py
@@ -84,7 +84,7 @@ def get_zone_capabilities(context):
 def select(context, specs=None):
     """Returns a list of hosts."""
     return _call_scheduler('select', context=context,
-            params={"specs": specs})
+            params={"request_spec": specs})
 
 
 def update_service_capabilities(context, service_name, host, capabilities):
diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py
index bd40e73c0..a29703aaf 100644
--- a/nova/scheduler/manager.py
+++ b/nova/scheduler/manager.py
@@ -70,6 +70,14 @@ class SchedulerManager(manager.Manager):
         self.zone_manager.update_service_capabilities(service_name,
                             host, capabilities)
 
+    def select(self, context=None, *args, **kwargs):
+        """Select a list of hosts best matching the provided specs."""
+        return self.driver.select(context, *args, **kwargs)
+
+    def get_scheduler_rules(self, context=None, *args, **kwargs):
+        """Ask the driver how requests should be made of it."""
+        return self.driver.get_scheduler_rules(context, *args, **kwargs)
+
     def _schedule(self, method, context, topic, *args, **kwargs):
         """Tries to call schedule_* method on the driver to retrieve host.
 
@@ -80,7 +88,9 @@ class SchedulerManager(manager.Manager):
         try:
             host = getattr(self.driver, driver_method)(elevated, *args,
                                                        **kwargs)
-        except AttributeError:
+        except AttributeError, e:
+            LOG.exception(_("Driver Method %(driver_method)s missing: %(e)s")
+                                % locals())
             host = self.driver.schedule(elevated, topic, *args, **kwargs)
 
         if not host:
diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py
index df84cf7bd..faa969124 100644
--- a/nova/scheduler/zone_aware_scheduler.py
+++ b/nova/scheduler/zone_aware_scheduler.py
@@ -21,16 +21,30 @@ across zones. There are two expansion points to this class for:
 """
 
 import operator
+import json
 
+import M2Crypto
+import novaclient
+
+from nova import crypto
 from nova import db
+from nova import exception
+from nova import flags
 from nova import log as logging
 from nova import rpc
+
 from nova.scheduler import api
 from nova.scheduler import driver
 
+FLAGS = flags.FLAGS
 LOG = logging.getLogger('nova.scheduler.zone_aware_scheduler')
 
 
+class InvalidBlob(exception.NovaException):
+    message = _("Ill-formed or incorrectly routed 'blob' data sent "
+                "to instance create request.")
+
+
 class ZoneAwareScheduler(driver.Scheduler):
     """Base class for creating Zone Aware Schedulers."""
 
@@ -38,6 +52,112 @@ class ZoneAwareScheduler(driver.Scheduler):
         """Call novaclient zone method. Broken out for testing."""
         return api.call_zone_method(context, method, specs=specs)
 
+    def _provision_resource_locally(self, context, item, instance_id, kwargs):
+        """Create the requested resource in this Zone."""
+        host = item['hostname']
+        kwargs['instance_id'] = instance_id
+        rpc.cast(context,
+                 db.queue_get_for(context, "compute", host),
+                 {"method": "run_instance",
+                  "args": kwargs})
+        LOG.debug(_("Provisioning locally via compute node %(host)s")
+                            % locals())
+
+    def _decrypt_blob(self, blob):
+        """Returns the decrypted blob or None if invalid. Broken out
+        for testing."""
+        decryptor = crypto.decryptor(FLAGS.build_plan_encryption_key)
+        try:
+            json_entry = decryptor(blob)
+            return json.dumps(entry)
+        except M2Crypto.EVP.EVPError:
+            pass
+        return None
+
+    def _ask_child_zone_to_create_instance(self, context, zone_info,
+                                           request_spec, kwargs):
+        """Once we have determined that the request should go to one
+        of our children, we need to fabricate a new POST /servers/
+        call with the same parameters that were passed into us.
+
+        Note that we have to reverse engineer from our args to get back the
+        image, flavor, ipgroup, etc. since the original call could have
+        come in from EC2 (which doesn't use these things)."""
+
+        instance_type = request_spec['instance_type']
+        instance_properties = request_spec['instance_properties']
+
+        name = instance_properties['display_name']
+        image_id = instance_properties['image_id']
+        meta = instance_properties['metadata']
+        flavor_id = instance_type['flavorid']
+
+        files = kwargs['injected_files']
+        ipgroup = None  # Not supported in OS API ... yet
+
+        child_zone = zone_info['child_zone']
+        child_blob = zone_info['child_blob']
+        zone = db.zone_get(context, child_zone)
+        url = zone.api_url
+        LOG.debug(_("Forwarding instance create call to child zone %(url)s")
+                    % locals())
+        nova = None
+        try:
+            nova = novaclient.OpenStack(zone.username, zone.password, url)
+            nova.authenticate()
+        except novaclient.exceptions.BadRequest, e:
+            raise exception.NotAuthorized(_("Bad credentials attempting "
+                            "to talk to zone at %(url)s.") % locals())
+
+        nova.servers.create(name, image_id, flavor_id, ipgroup, meta, files,
+                            child_blob)
+
+    def _provision_resource_from_blob(self, context, item, instance_id,
+                                          request_spec, kwargs):
+        """Create the requested resource locally or in a child zone
+           based on what is stored in the zone blob info.
+
+           Attempt to decrypt the blob to see if this request is:
+           1. valid, and
+           2. intended for this zone or a child zone.
+
+           Note: If we have "blob" that means the request was passed
+           into us from a parent zone. If we have "child_blob" that
+           means we gathered the info from one of our children.
+           It's possible that, when we decrypt the 'blob' field, it
+           contains "child_blob" data. In which case we forward the
+           request."""
+
+        host_info = None
+        if "blob" in item:
+            # Request was passed in from above. Is it for us?
+            host_info = self._decrypt_blob(item['blob'])
+        elif "child_blob" in item:
+            # Our immediate child zone provided this info ...
+            host_info = item
+
+        if not host_info:
+            raise InvalidBlob()
+
+        # Valid data ... is it for us?
+        if 'child_zone' in host_info and 'child_blob' in host_info:
+            self._ask_child_zone_to_create_instance(context, host_info,
+                                                    request_spec, kwargs)
+        else:
+            self._provision_resource_locally(context, host_info,
+                                             instance_id, kwargs)
+
+    def _provision_resource(self, context, item, instance_id, request_spec,
+                           kwargs):
+        """Create the requested resource in this Zone or a child zone."""
+        if "hostname" in item:
+            self._provision_resource_locally(context, item, instance_id,
+                            kwargs)
+            return
+
+        self._provision_resource_from_blob(context, item, instance_id,
+                                               request_spec, kwargs)
+
     def schedule_run_instance(self, context, instance_id, request_spec,
                               *args, **kwargs):
         """This method is called from nova.compute.api to provision
@@ -51,8 +171,10 @@ class ZoneAwareScheduler(driver.Scheduler):
 
         # TODO(sandy): We'll have to look for richer specs at some point.
 
-        if 'blob' in request_spec:
-            self.provision_resource(context, request_spec, instance_id, kwargs)
+        blob = request_spec.get('blob')
+        if blob:
+            self._provision_resource(context, request_spec, instance_id,
+                                    request_spec, kwargs)
             return None
 
         # Create build plan and provision ...
@@ -61,28 +183,13 @@ class ZoneAwareScheduler(driver.Scheduler):
             raise driver.NoValidHost(_('No hosts were available'))
 
         for item in build_plan:
-            self.provision_resource(context, item, instance_id, kwargs)
+            self._provision_resource(context, item, instance_id, request_spec,
+                                    kwargs)
 
         # Returning None short-circuits the routing to Compute (since
         # we've already done it here)
         return None
 
-    def provision_resource(self, context, item, instance_id, kwargs):
-        """Create the requested resource in this Zone or a child zone."""
-        if "hostname" in item:
-            host = item['hostname']
-            kwargs['instance_id'] = instance_id
-            rpc.cast(context,
-                     db.queue_get_for(context, "compute", host),
-                     {"method": "run_instance",
-                      "args": kwargs})
-            LOG.debug(_("Casted to compute %(host)s for run_instance")
-                                % locals())
-        else:
-            # TODO(sandy) Provision in child zone ...
-            LOG.warning(_("Provision to Child Zone not supported (yet)"))
-            pass
-
     def select(self, context, request_spec, *args, **kwargs):
         """Select returns a list of weights and zone/host information
         corresponding to the best hosts to service the request. Any
@@ -124,17 +231,17 @@ class ZoneAwareScheduler(driver.Scheduler):
         weighted = self.weigh_hosts(num_instances, request_spec, host_list)
 
         # Next, tack on the best weights from the child zones ...
+        json_spec = json.dumps(request_spec)
         child_results = self._call_zone_method(context, "select",
-                specs=request_spec)
+                specs=json_spec)
         for child_zone, result in child_results:
             for weighting in result:
                 # Remember the child_zone so we can get back to
                 # it later if needed. This implicitly builds a zone
                 # path structure.
-                host_dict = {
-                        "weight": weighting["weight"],
-                        "child_zone": child_zone,
-                        "child_blob": weighting["blob"]}
+                host_dict = {"weight": weighting["weight"],
+                             "child_zone": child_zone,
+                             "child_blob": weighting["blob"]}
                 weighted.append(host_dict)
 
         weighted.sort(key=operator.itemgetter('weight'))
diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py
index fa2e05033..098577e4c 100644
--- a/nova/tests/api/openstack/test_zones.py
+++ b/nova/tests/api/openstack/test_zones.py
@@ -21,7 +21,6 @@ import json
 import nova.db
 from nova import context
 from nova import crypto
-from nova import exception
 from nova import flags
 from nova import test
 from nova.api.openstack import zones
@@ -210,6 +209,11 @@ class ZonesTest(test.TestCase):
         self.stubs.Set(api, 'select', zone_select)
 
         req = webob.Request.blank('/v1.0/zones/select')
+        req.method = 'POST'
+        req.headers["Content-Type"] = "application/json"
+        # Select queries end up being JSON encoded twice.
+        # Once to a string and again as an HTTP POST Body
+        req.body = json.dumps(json.dumps({}))
 
         res = req.get_response(fakes.wsgi_app())
         res_dict = json.loads(res.body)
diff --git a/nova/tests/scheduler/test_zone_aware_scheduler.py b/nova/tests/scheduler/test_zone_aware_scheduler.py
index 561fdea94..9f70b9dbc 100644
--- a/nova/tests/scheduler/test_zone_aware_scheduler.py
+++ b/nova/tests/scheduler/test_zone_aware_scheduler.py
@@ -16,6 +16,7 @@
 Tests For Zone Aware Scheduler.
 """
 
+from nova import exception
 from nova import test
 from nova.scheduler import driver
 from nova.scheduler import zone_aware_scheduler
@@ -90,6 +91,41 @@ def fake_empty_call_zone_method(context, method, specs):
     return []
 
 
+# Hmm, I should probably be using mox for this.
+was_called = False
+
+
+def fake_provision_resource(context, item, instance_id, request_spec, kwargs):
+    global was_called
+    was_called = True
+
+
+def fake_ask_child_zone_to_create_instance(context, zone_info,
+                                           request_spec, kwargs):
+    global was_called
+    was_called = True
+
+
+def fake_provision_resource_locally(context, item, instance_id, kwargs):
+    global was_called
+    was_called = True
+
+
+def fake_provision_resource_from_blob(context, item, instance_id,
+                                      request_spec, kwargs):
+    global was_called
+    was_called = True
+
+
+def fake_decrypt_blob_returns_local_info(blob):
+    return {'foo': True}  # values aren't important.
+
+
+def fake_decrypt_blob_returns_child_info(blob):
+    return {'child_zone': True,
+            'child_blob': True}  # values aren't important. Keys are.
+
+
 def fake_call_zone_method(context, method, specs):
     return [
         ('zone1', [
@@ -149,4 +185,112 @@ class ZoneAwareSchedulerTestCase(test.TestCase):
         fake_context = {}
         self.assertRaises(driver.NoValidHost, sched.schedule_run_instance,
                           fake_context, 1,
-                          dict(host_filter=None, instance_type={}))
+                          dict(host_filter=None,
+                               request_spec={'instance_type': {}}))
+
+    def test_schedule_do_not_schedule_with_hint(self):
+        """
+        Check the local/child zone routing in the run_instance() call.
+        If the zone_blob hint was passed in, don't re-schedule.
+        """
+        global was_called
+        sched = FakeZoneAwareScheduler()
+        was_called = False
+        self.stubs.Set(sched, '_provision_resource', fake_provision_resource)
+        request_spec = {
+                'instance_properties': {},
+                'instance_type': {},
+                'filter_driver': 'nova.scheduler.host_filter.AllHostsFilter',
+                'blob': "Non-None blob data"
+            }
+
+        result = sched.schedule_run_instance(None, 1, request_spec)
+        self.assertEquals(None, result)
+        self.assertTrue(was_called)
+
+    def test_provision_resource_local(self):
+        """Provision a resource locally or remotely."""
+        global was_called
+        sched = FakeZoneAwareScheduler()
+        was_called = False
+        self.stubs.Set(sched, '_provision_resource_locally',
+                       fake_provision_resource_locally)
+
+        request_spec = {'hostname': "foo"}
+        sched._provision_resource(None, request_spec, 1, request_spec, {})
+        self.assertTrue(was_called)
+
+    def test_provision_resource_remote(self):
+        """Provision a resource locally or remotely."""
+        global was_called
+        sched = FakeZoneAwareScheduler()
+        was_called = False
+        self.stubs.Set(sched, '_provision_resource_from_blob',
+                       fake_provision_resource_from_blob)
+
+        request_spec = {}
+        sched._provision_resource(None, request_spec, 1, request_spec, {})
+        self.assertTrue(was_called)
+
+    def test_provision_resource_from_blob_empty(self):
+        """Provision a resource locally or remotely given no hints."""
+        global was_called
+        sched = FakeZoneAwareScheduler()
+        request_spec = {}
+        self.assertRaises(zone_aware_scheduler.InvalidBlob,
+                          sched._provision_resource_from_blob,
+                          None, {}, 1, {}, {})
+
+    def test_provision_resource_from_blob_with_local_blob(self):
+        """
+        Provision a resource locally or remotely when blob hint passed in.
+        """
+        global was_called
+        sched = FakeZoneAwareScheduler()
+        was_called = False
+        self.stubs.Set(sched, '_decrypt_blob',
+                       fake_decrypt_blob_returns_local_info)
+        self.stubs.Set(sched, '_provision_resource_locally',
+                       fake_provision_resource_locally)
+
+        request_spec = {'blob': "Non-None blob data"}
+
+        sched._provision_resource_from_blob(None, request_spec, 1,
+                                            request_spec, {})
+        self.assertTrue(was_called)
+
+    def test_provision_resource_from_blob_with_child_blob(self):
+        """
+        Provision a resource locally or remotely when child blob hint
+        passed in.
+        """
+        global was_called
+        sched = FakeZoneAwareScheduler()
+        self.stubs.Set(sched, '_decrypt_blob',
+                       fake_decrypt_blob_returns_child_info)
+        was_called = False
+        self.stubs.Set(sched, '_ask_child_zone_to_create_instance',
+                       fake_ask_child_zone_to_create_instance)
+
+        request_spec = {'blob': "Non-None blob data"}
+
+        sched._provision_resource_from_blob(None, request_spec, 1,
+                                            request_spec, {})
+        self.assertTrue(was_called)
+
+    def test_provision_resource_from_blob_with_immediate_child_blob(self):
+        """
+        Provision a resource locally or remotely when blob hint passed in
+        from an immediate child.
+        """
+        global was_called
+        sched = FakeZoneAwareScheduler()
+        was_called = False
+        self.stubs.Set(sched, '_ask_child_zone_to_create_instance',
+                       fake_ask_child_zone_to_create_instance)
+
+        request_spec = {'child_blob': True, 'child_zone': True}
+
+        sched._provision_resource_from_blob(None, request_spec, 1,
+                                            request_spec, {})
+        self.assertTrue(was_called)