Merged trunk

10 files changed, 400 insertions, 28 deletions

diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py
index 1b4f28f59..0dad46268 100644
--- a/nova/api/openstack/servers.py
+++ b/nova/api/openstack/servers.py
@@ -37,6 +37,7 @@ from nova.auth import manager as auth_manager
 from nova.compute import instance_types
 from nova.compute import power_state
 import nova.api.openstack
+from nova.scheduler import api as scheduler_api
 
 
 LOG = logging.getLogger('server')
@@ -87,15 +88,18 @@ class Controller(wsgi.Controller):
                 for inst in limited_list]
         return dict(servers=servers)
 
+    @scheduler_api.redirect_handler
     def show(self, req, id):
         """ Returns server details by server id """
         try:
-            instance = self.compute_api.get(req.environ['nova.context'], id)
+            instance = self.compute_api.routing_get(
+                req.environ['nova.context'], id)
             builder = self._get_view_builder(req)
             return builder.build(instance, is_detail=True)
         except exception.NotFound:
             return faults.Fault(exc.HTTPNotFound())
 
+    @scheduler_api.redirect_handler
     def delete(self, req, id):
         """ Destroys a server """
         try:
@@ -226,6 +230,7 @@ class Controller(wsgi.Controller):
         # if the original error is okay, just reraise it
         raise error
 
+    @scheduler_api.redirect_handler
     def update(self, req, id):
         """ Updates the server name or password """
         if len(req.body) == 0:
@@ -251,6 +256,7 @@ class Controller(wsgi.Controller):
             return faults.Fault(exc.HTTPNotFound())
         return exc.HTTPNoContent()
 
+    @scheduler_api.redirect_handler
     def action(self, req, id):
         """Multi-purpose method used to reboot, rebuild, or
         resize a server"""
@@ -316,6 +322,7 @@ class Controller(wsgi.Controller):
             return faults.Fault(exc.HTTPUnprocessableEntity())
         return exc.HTTPAccepted()
 
+    @scheduler_api.redirect_handler
     def lock(self, req, id):
         """
         lock the instance with id
@@ -331,6 +338,7 @@ class Controller(wsgi.Controller):
             return faults.Fault(exc.HTTPUnprocessableEntity())
         return exc.HTTPAccepted()
 
+    @scheduler_api.redirect_handler
     def unlock(self, req, id):
         """
         unlock the instance with id
@@ -346,6 +354,7 @@ class Controller(wsgi.Controller):
             return faults.Fault(exc.HTTPUnprocessableEntity())
         return exc.HTTPAccepted()
 
+    @scheduler_api.redirect_handler
     def get_lock(self, req, id):
         """
         return the boolean state of (instance with id)'s lock
@@ -360,6 +369,7 @@ class Controller(wsgi.Controller):
             return faults.Fault(exc.HTTPUnprocessableEntity())
         return exc.HTTPAccepted()
 
+    @scheduler_api.redirect_handler
     def reset_network(self, req, id):
         """
         Reset networking on an instance (admin only).
@@ -374,6 +384,7 @@ class Controller(wsgi.Controller):
             return faults.Fault(exc.HTTPUnprocessableEntity())
         return exc.HTTPAccepted()
 
+    @scheduler_api.redirect_handler
     def inject_network_info(self, req, id):
         """
         Inject network info for an instance (admin only).
@@ -388,6 +399,7 @@ class Controller(wsgi.Controller):
             return faults.Fault(exc.HTTPUnprocessableEntity())
         return exc.HTTPAccepted()
 
+    @scheduler_api.redirect_handler
     def pause(self, req, id):
         """ Permit Admins to Pause the server. """
         ctxt = req.environ['nova.context']
@@ -399,6 +411,7 @@ class Controller(wsgi.Controller):
             return faults.Fault(exc.HTTPUnprocessableEntity())
         return exc.HTTPAccepted()
 
+    @scheduler_api.redirect_handler
     def unpause(self, req, id):
         """ Permit Admins to Unpause the server. """
         ctxt = req.environ['nova.context']
@@ -410,6 +423,7 @@ class Controller(wsgi.Controller):
             return faults.Fault(exc.HTTPUnprocessableEntity())
         return exc.HTTPAccepted()
 
+    @scheduler_api.redirect_handler
     def suspend(self, req, id):
         """permit admins to suspend the server"""
         context = req.environ['nova.context']
@@ -421,6 +435,7 @@ class Controller(wsgi.Controller):
             return faults.Fault(exc.HTTPUnprocessableEntity())
         return exc.HTTPAccepted()
 
+    @scheduler_api.redirect_handler
     def resume(self, req, id):
         """permit admins to resume the server from suspend"""
         context = req.environ['nova.context']
@@ -432,6 +447,7 @@ class Controller(wsgi.Controller):
             return faults.Fault(exc.HTTPUnprocessableEntity())
         return exc.HTTPAccepted()
 
+    @scheduler_api.redirect_handler
     def rescue(self, req, id):
         """Permit users to rescue the server."""
         context = req.environ["nova.context"]
@@ -443,6 +459,7 @@ class Controller(wsgi.Controller):
             return faults.Fault(exc.HTTPUnprocessableEntity())
         return exc.HTTPAccepted()
 
+    @scheduler_api.redirect_handler
     def unrescue(self, req, id):
         """Permit users to unrescue the server."""
         context = req.environ["nova.context"]
@@ -454,6 +471,7 @@ class Controller(wsgi.Controller):
             return faults.Fault(exc.HTTPUnprocessableEntity())
         return exc.HTTPAccepted()
 
+    @scheduler_api.redirect_handler
     def get_ajax_console(self, req, id):
         """ Returns a url to an instance's ajaxterm console. """
         try:
@@ -463,6 +481,7 @@ class Controller(wsgi.Controller):
             return faults.Fault(exc.HTTPNotFound())
         return exc.HTTPAccepted()
 
+    @scheduler_api.redirect_handler
     def diagnostics(self, req, id):
         """Permit Admins to retrieve server diagnostics."""
         ctxt = req.environ["nova.context"]
diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py
index d4a59993b..846cb48a1 100644
--- a/nova/api/openstack/zones.py
+++ b/nova/api/openstack/zones.py
@@ -17,6 +17,7 @@ import common
 
 from nova import db
 from nova import flags
+from nova import log as logging
 from nova import wsgi
 from nova.scheduler import api
 
@@ -38,7 +39,8 @@ def _exclude_keys(item, keys):
 
 
 def _scrub_zone(zone):
-    return _filter_keys(zone, ('id', 'api_url'))
+    return _exclude_keys(zone, ('username', 'password', 'created_at',
+                    'deleted', 'deleted_at', 'updated_at'))
 
 
 class Controller(wsgi.Controller):
@@ -53,12 +55,8 @@ class Controller(wsgi.Controller):
         # Ask the ZoneManager in the Scheduler for most recent data,
         # or fall-back to the database ...
         items = api.get_zone_list(req.environ['nova.context'])
-        if not items:
-            items = db.zone_get_all(req.environ['nova.context'])
-
         items = common.limited(items, req)
-        items = [_exclude_keys(item, ['username', 'password'])
-                      for item in items]
+        items = [_scrub_zone(item) for item in items]
         return dict(zones=items)
 
     def detail(self, req):
@@ -81,23 +79,23 @@ class Controller(wsgi.Controller):
     def show(self, req, id):
         """Return data about the given zone id"""
         zone_id = int(id)
-        zone = db.zone_get(req.environ['nova.context'], zone_id)
+        zone = api.zone_get(req.environ['nova.context'], zone_id)
         return dict(zone=_scrub_zone(zone))
 
     def delete(self, req, id):
         zone_id = int(id)
-        db.zone_delete(req.environ['nova.context'], zone_id)
+        api.zone_delete(req.environ['nova.context'], zone_id)
         return {}
 
     def create(self, req):
         context = req.environ['nova.context']
         env = self._deserialize(req.body, req.get_content_type())
-        zone = db.zone_create(context, env["zone"])
+        zone = api.zone_create(context, env["zone"])
         return dict(zone=_scrub_zone(zone))
 
     def update(self, req, id):
         context = req.environ['nova.context']
         env = self._deserialize(req.body, req.get_content_type())
         zone_id = int(id)
-        zone = db.zone_update(context, zone_id, env["zone"])
+        zone = api.zone_update(context, zone_id, env["zone"])
         return dict(zone=_scrub_zone(zone))
diff --git a/nova/compute/api.py b/nova/compute/api.py
index 01eead4ac..309847156 100644
--- a/nova/compute/api.py
+++ b/nova/compute/api.py
@@ -34,6 +34,7 @@ from nova import rpc
 from nova import utils
 from nova import volume
 from nova.compute import instance_types
+from nova.scheduler import api as scheduler_api
 from nova.db import base
 
 FLAGS = flags.FLAGS
@@ -352,6 +353,7 @@ class API(base.Base):
         rv = self.db.instance_update(context, instance_id, kwargs)
         return dict(rv.iteritems())
 
+    @scheduler_api.reroute_compute("delete")
     def delete(self, context, instance_id):
         LOG.debug(_("Going to try to terminate %s"), instance_id)
         try:
@@ -384,6 +386,13 @@ class API(base.Base):
         rv = self.db.instance_get(context, instance_id)
         return dict(rv.iteritems())
 
+    @scheduler_api.reroute_compute("get")
+    def routing_get(self, context, instance_id):
+        """Use this method instead of get() if this is the only
+           operation you intend to to. It will route to novaclient.get
+           if the instance is not found."""
+        return self.get(context, instance_id)
+
     def get_all(self, context, project_id=None, reservation_id=None,
                 fixed_ip=None):
         """Get all instances, possibly filtered by one of the
@@ -527,14 +536,17 @@ class API(base.Base):
                               "instance_id": instance_id,
                               "flavor_id": flavor_id}})
 
+    @scheduler_api.reroute_compute("pause")
     def pause(self, context, instance_id):
         """Pause the given instance."""
         self._cast_compute_message('pause_instance', context, instance_id)
 
+    @scheduler_api.reroute_compute("unpause")
     def unpause(self, context, instance_id):
         """Unpause the given instance."""
         self._cast_compute_message('unpause_instance', context, instance_id)
 
+    @scheduler_api.reroute_compute("diagnostics")
     def get_diagnostics(self, context, instance_id):
         """Retrieve diagnostics for the given instance."""
         return self._call_compute_message(
@@ -546,18 +558,22 @@ class API(base.Base):
         """Retrieve actions for the given instance."""
         return self.db.instance_get_actions(context, instance_id)
 
+    @scheduler_api.reroute_compute("suspend")
     def suspend(self, context, instance_id):
         """suspend the instance with instance_id"""
         self._cast_compute_message('suspend_instance', context, instance_id)
 
+    @scheduler_api.reroute_compute("resume")
     def resume(self, context, instance_id):
         """resume the instance with instance_id"""
         self._cast_compute_message('resume_instance', context, instance_id)
 
+    @scheduler_api.reroute_compute("rescue")
     def rescue(self, context, instance_id):
         """Rescue the given instance."""
         self._cast_compute_message('rescue_instance', context, instance_id)
 
+    @scheduler_api.reroute_compute("unrescue")
     def unrescue(self, context, instance_id):
         """Unrescue the given instance."""
         self._cast_compute_message('unrescue_instance', context, instance_id)
@@ -573,7 +589,6 @@ class API(base.Base):
 
     def get_ajax_console(self, context, instance_id):
         """Get a url to an AJAX Console"""
-        instance = self.get(context, instance_id)
         output = self._call_compute_message('get_ajax_console',
                                             context,
                                             instance_id)
diff --git a/nova/db/api.py b/nova/db/api.py
index caa719bbc..91e7cbf31 100644
--- a/nova/db/api.py
+++ b/nova/db/api.py
@@ -71,6 +71,7 @@ class NoMoreTargets(exception.Error):
     """No more available blades"""
     pass
 
+
 ###################
 
 
diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py
index 796d6ba31..06b05366a 100644
--- a/nova/network/linux_net.py
+++ b/nova/network/linux_net.py
@@ -1,3 +1,5 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
 # Copyright 2010 United States Government as represented by the
 # Administrator of the National Aeronautics and Space Administration.
 # All Rights Reserved.
@@ -210,10 +212,7 @@ class IptablesManager(object):
     """
     def __init__(self, execute=None):
         if not execute:
-            if FLAGS.fake_network:
-                self.execute = lambda *args, **kwargs: ('', '')
-            else:
-                self.execute = utils.execute
+            self.execute = _execute
         else:
             self.execute = execute
 
@@ -352,9 +351,6 @@ class IptablesManager(object):
         return new_filter
 
 
-iptables_manager = IptablesManager()
-
-
 def metadata_forward():
     """Create forwarding rule for metadata"""
     iptables_manager.ipv4['nat'].add_rule("PREROUTING",
@@ -767,3 +763,6 @@ def _ip_bridge_cmd(action, params, device):
     cmd.extend(params)
     cmd.extend(['dev', device])
     return cmd
+
+
+iptables_manager = IptablesManager()
diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py
index 19a05b716..6bb3bf3cd 100644
--- a/nova/scheduler/api.py
+++ b/nova/scheduler/api.py
@@ -17,11 +17,21 @@
 Handles all requests relating to schedulers.
 """
 
+import novaclient
+
+from nova import db
+from nova import exception
 from nova import flags
 from nova import log as logging
 from nova import rpc
 
+from eventlet import greenpool
+
 FLAGS = flags.FLAGS
+flags.DEFINE_bool('enable_zone_routing',
+    False,
+    'When True, routing to child zones will occur.')
+
 LOG = logging.getLogger('nova.scheduler.api')
 
 
@@ -45,9 +55,27 @@ def get_zone_list(context):
     items = _call_scheduler('get_zone_list', context)
     for item in items:
         item['api_url'] = item['api_url'].replace('\\/', '/')
+    if not items:
+        items = db.zone_get_all(context)
     return items
 
 
+def zone_get(context, zone_id):
+    return db.zone_get(context, zone_id)
+
+
+def zone_delete(context, zone_id):
+    return db.zone_delete(context, zone_id)
+
+
+def zone_create(context, data):
+    return db.zone_create(context, data)
+
+
+def zone_update(context, zone_id, data):
+    return db.zone_update(context, zone_id, data)
+
+
 def get_zone_capabilities(context, service=None):
     """Returns a dict of key, value capabilities for this zone,
        or for a particular class of services running in this zone."""
@@ -62,3 +90,152 @@ def update_service_capabilities(context, service_name, host, capabilities):
                   args=dict(service_name=service_name, host=host,
                             capabilities=capabilities))
     return rpc.fanout_cast(context, 'scheduler', kwargs)
+
+
+def _wrap_method(function, self):
+    """Wrap method to supply self."""
+    def _wrap(*args, **kwargs):
+        return function(self, *args, **kwargs)
+    return _wrap
+
+
+def _process(func, zone):
+    """Worker stub for green thread pool. Give the worker
+    an authenticated nova client and zone info."""
+    nova = novaclient.OpenStack(zone.username, zone.password, zone.api_url)
+    nova.authenticate()
+    return func(nova, zone)
+
+
+def child_zone_helper(zone_list, func):
+    """Fire off a command to each zone in the list.
+    The return is [novaclient return objects] from each child zone.
+    For example, if you are calling server.pause(), the list will
+    be whatever the response from server.pause() is. One entry
+    per child zone called."""
+    green_pool = greenpool.GreenPool()
+    return [result for result in green_pool.imap(
+                    _wrap_method(_process, func), zone_list)]
+
+
+def _issue_novaclient_command(nova, zone, collection, method_name, item_id):
+    """Use novaclient to issue command to a single child zone.
+       One of these will be run in parallel for each child zone."""
+    manager = getattr(nova, collection)
+    result = None
+    try:
+        try:
+            result = manager.get(int(item_id))
+        except ValueError, e:
+            result = manager.find(name=item_id)
+    except novaclient.NotFound:
+        url = zone.api_url
+        LOG.debug(_("%(collection)s '%(item_id)s' not found on '%(url)s'" %
+                                                locals()))
+        return None
+
+    if method_name.lower() not in ['get', 'find']:
+        result = getattr(result, method_name)()
+    return result
+
+
+def wrap_novaclient_function(f, collection, method_name, item_id):
+    """Appends collection, method_name and item_id to the incoming
+    (nova, zone) call from child_zone_helper."""
+    def inner(nova, zone):
+        return f(nova, zone, collection, method_name, item_id)
+
+    return inner
+
+
+class RedirectResult(exception.Error):
+    """Used to the HTTP API know that these results are pre-cooked
+    and they can be returned to the caller directly."""
+    def __init__(self, results):
+        self.results = results
+        super(RedirectResult, self).__init__(
+               message=_("Uncaught Zone redirection exception"))
+
+
+class reroute_compute(object):
+    """Decorator used to indicate that the method should
+       delegate the call the child zones if the db query
+       can't find anything."""
+    def __init__(self, method_name):
+        self.method_name = method_name
+
+    def __call__(self, f):
+        def wrapped_f(*args, **kwargs):
+            collection, context, item_id = \
+                            self.get_collection_context_and_id(args, kwargs)
+            try:
+                # Call the original function ...
+                return f(*args, **kwargs)
+            except exception.InstanceNotFound, e:
+                LOG.debug(_("Instance %(item_id)s not found "
+                                    "locally: '%(e)s'" % locals()))
+
+                if not FLAGS.enable_zone_routing:
+                    raise
+
+                zones = db.zone_get_all(context)
+                if not zones:
+                    raise
+
+                # Ask the children to provide an answer ...
+                LOG.debug(_("Asking child zones ..."))
+                result = self._call_child_zones(zones,
+                            wrap_novaclient_function(_issue_novaclient_command,
+                                   collection, self.method_name, item_id))
+                # Scrub the results and raise another exception
+                # so the API layers can bail out gracefully ...
+                raise RedirectResult(self.unmarshall_result(result))
+        return wrapped_f
+
+    def _call_child_zones(self, zones, function):
+        """Ask the child zones to perform this operation.
+        Broken out for testing."""
+        return child_zone_helper(zones, function)
+
+    def get_collection_context_and_id(self, args, kwargs):
+        """Returns a tuple of (novaclient collection name, security
+           context and resource id. Derived class should override this."""
+        context = kwargs.get('context', None)
+        instance_id = kwargs.get('instance_id', None)
+        if len(args) > 0 and not context:
+            context = args[1]
+        if len(args) > 1 and not instance_id:
+            instance_id = args[2]
+        return ("servers", context, instance_id)
+
+    def unmarshall_result(self, zone_responses):
+        """Result is a list of responses from each child zone.
+        Each decorator derivation is responsible to turning this
+        into a format expected by the calling method. For
+        example, this one is expected to return a single Server
+        dict {'server':{k:v}}. Others may return a list of them, like
+        {'servers':[{k,v}]}"""
+        reduced_response = []
+        for zone_response in zone_responses:
+            if not zone_response:
+                continue
+
+            server = zone_response.__dict__
+
+            for k in server.keys():
+                if k[0] == '_' or k == 'manager':
+                    del server[k]
+
+            reduced_response.append(dict(server=server))
+        if reduced_response:
+            return reduced_response[0]  # first for now.
+        return {}
+
+
+def redirect_handler(f):
+    def new_f(*args, **kwargs):
+        try:
+            return f(*args, **kwargs)
+        except RedirectResult, e:
+            return e.results
+    return new_f
diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py
index c1a50dbc3..198f9d4cc 100644
--- a/nova/scheduler/zone_manager.py
+++ b/nova/scheduler/zone_manager.py
@@ -58,8 +58,9 @@ class ZoneState(object):
            child zone."""
         self.last_seen = datetime.now()
         self.attempt = 0
-        self.name = zone_metadata["name"]
-        self.capabilities = zone_metadata["capabilities"]
+        self.name = zone_metadata.get("name", "n/a")
+        self.capabilities = ", ".join(["%s=%s" % (k, v)
+                        for k, v in zone_metadata.iteritems() if k != 'name'])
         self.is_active = True
 
     def to_dict(self):
@@ -104,7 +105,7 @@ class ZoneManager(object):
     """Keeps the zone states updated."""
     def __init__(self):
         self.last_zone_db_check = datetime.min
-        self.zone_states = {}
+        self.zone_states = {}  # { <zone_id> : ZoneState }
         self.service_states = {}  # { <service> : { <host> : { cap k : v }}}
         self.green_pool = greenpool.GreenPool()
 
diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py
index 244e43bd9..6df74dd61 100644
--- a/nova/tests/test_scheduler.py
+++ b/nova/tests/test_scheduler.py
@@ -21,6 +21,9 @@ Tests For Scheduler
 
 import datetime
 import mox
+import novaclient.exceptions
+import stubout
+import webob
 
 from mox import IgnoreArg
 from nova import context
@@ -32,6 +35,7 @@ from nova import test
 from nova import rpc
 from nova import utils
 from nova.auth import manager as auth_manager
+from nova.scheduler import api
 from nova.scheduler import manager
 from nova.scheduler import driver
 from nova.compute import power_state
@@ -937,3 +941,160 @@ class SimpleDriverTestCase(test.TestCase):
         db.instance_destroy(self.context, instance_id)
         db.service_destroy(self.context, s_ref['id'])
         db.service_destroy(self.context, s_ref2['id'])
+
+
+class FakeZone(object):
+    def __init__(self, api_url, username, password):
+        self.api_url = api_url
+        self.username = username
+        self.password = password
+
+
+def zone_get_all(context):
+    return [
+                FakeZone('http://example.com', 'bob', 'xxx'),
+           ]
+
+
+class FakeRerouteCompute(api.reroute_compute):
+    def _call_child_zones(self, zones, function):
+        return []
+
+    def get_collection_context_and_id(self, args, kwargs):
+        return ("servers", None, 1)
+
+    def unmarshall_result(self, zone_responses):
+        return dict(magic="found me")
+
+
+def go_boom(self, context, instance):
+    raise exception.InstanceNotFound("boom message", instance)
+
+
+def found_instance(self, context, instance):
+    return dict(name='myserver')
+
+
+class FakeResource(object):
+    def __init__(self, attribute_dict):
+        for k, v in attribute_dict.iteritems():
+            setattr(self, k, v)
+
+    def pause(self):
+        pass
+
+
+class ZoneRedirectTest(test.TestCase):
+    def setUp(self):
+        super(ZoneRedirectTest, self).setUp()
+        self.stubs = stubout.StubOutForTesting()
+
+        self.stubs.Set(db, 'zone_get_all', zone_get_all)
+
+        self.enable_zone_routing = FLAGS.enable_zone_routing
+        FLAGS.enable_zone_routing = True
+
+    def tearDown(self):
+        self.stubs.UnsetAll()
+        FLAGS.enable_zone_routing = self.enable_zone_routing
+        super(ZoneRedirectTest, self).tearDown()
+
+    def test_trap_found_locally(self):
+        decorator = FakeRerouteCompute("foo")
+        try:
+            result = decorator(found_instance)(None, None, 1)
+        except api.RedirectResult, e:
+            self.fail(_("Successful database hit should succeed"))
+
+    def test_trap_not_found_locally(self):
+        decorator = FakeRerouteCompute("foo")
+        try:
+            result = decorator(go_boom)(None, None, 1)
+            self.assertFail(_("Should have rerouted."))
+        except api.RedirectResult, e:
+            self.assertEquals(e.results['magic'], 'found me')
+
+    def test_routing_flags(self):
+        FLAGS.enable_zone_routing = False
+        decorator = FakeRerouteCompute("foo")
+        try:
+            result = decorator(go_boom)(None, None, 1)
+            self.assertFail(_("Should have thrown exception."))
+        except exception.InstanceNotFound, e:
+            self.assertEquals(e.message, 'boom message')
+
+    def test_get_collection_context_and_id(self):
+        decorator = api.reroute_compute("foo")
+        self.assertEquals(decorator.get_collection_context_and_id(
+            (None, 10, 20), {}), ("servers", 10, 20))
+        self.assertEquals(decorator.get_collection_context_and_id(
+            (None, 11,),  dict(instance_id=21)), ("servers", 11, 21))
+        self.assertEquals(decorator.get_collection_context_and_id(
+            (None,), dict(context=12, instance_id=22)), ("servers", 12, 22))
+
+    def test_unmarshal_single_server(self):
+        decorator = api.reroute_compute("foo")
+        self.assertEquals(decorator.unmarshall_result([]), {})
+        self.assertEquals(decorator.unmarshall_result(
+                [FakeResource(dict(a=1, b=2)), ]),
+                dict(server=dict(a=1, b=2)))
+        self.assertEquals(decorator.unmarshall_result(
+                [FakeResource(dict(a=1, _b=2)), ]),
+                dict(server=dict(a=1,)))
+        self.assertEquals(decorator.unmarshall_result(
+                [FakeResource(dict(a=1, manager=2)), ]),
+                dict(server=dict(a=1,)))
+        self.assertEquals(decorator.unmarshall_result(
+                [FakeResource(dict(_a=1, manager=2)), ]),
+                dict(server={}))
+
+
+class FakeServerCollection(object):
+    def get(self, instance_id):
+        return FakeResource(dict(a=10, b=20))
+
+    def find(self, name):
+        return FakeResource(dict(a=11, b=22))
+
+
+class FakeEmptyServerCollection(object):
+    def get(self, f):
+        raise novaclient.NotFound(1)
+
+    def find(self, name):
+        raise novaclient.NotFound(2)
+
+
+class FakeNovaClient(object):
+    def __init__(self, collection):
+        self.servers = collection
+
+
+class DynamicNovaClientTest(test.TestCase):
+    def test_issue_novaclient_command_found(self):
+        zone = FakeZone('http://example.com', 'bob', 'xxx')
+        self.assertEquals(api._issue_novaclient_command(
+                    FakeNovaClient(FakeServerCollection()),
+                    zone, "servers", "get", 100).a, 10)
+
+        self.assertEquals(api._issue_novaclient_command(
+                    FakeNovaClient(FakeServerCollection()),
+                    zone, "servers", "find", "name").b, 22)
+
+        self.assertEquals(api._issue_novaclient_command(
+                    FakeNovaClient(FakeServerCollection()),
+                    zone, "servers", "pause", 100), None)
+
+    def test_issue_novaclient_command_not_found(self):
+        zone = FakeZone('http://example.com', 'bob', 'xxx')
+        self.assertEquals(api._issue_novaclient_command(
+                    FakeNovaClient(FakeEmptyServerCollection()),
+                    zone, "servers", "get", 100), None)
+
+        self.assertEquals(api._issue_novaclient_command(
+                    FakeNovaClient(FakeEmptyServerCollection()),
+                    zone, "servers", "find", "name"), None)
+
+        self.assertEquals(api._issue_novaclient_command(
+                    FakeNovaClient(FakeEmptyServerCollection()),
+                    zone, "servers", "any", "name"), None)
diff --git a/nova/tests/test_volume.py b/nova/tests/test_volume.py
index 5d68ca2ae..d71b75f3f 100644
--- a/nova/tests/test_volume.py
+++ b/nova/tests/test_volume.py
@@ -356,8 +356,8 @@ class ISCSITestCase(DriverTestCase):
         tid = db.volume_get_iscsi_target_num(self.context, volume_id_list[0])
         self.mox.StubOutWithMock(self.volume.driver, '_execute')
         self.volume.driver._execute("sudo", "ietadm", "--op", "show",
-                                    "--tid=%(tid)d" % locals()
-                                 ).AndRaise(exception.ProcessExecutionError())
+                                    "--tid=%(tid)d" % locals()).AndRaise(
+                                            exception.ProcessExecutionError())
 
         self.mox.ReplayAll()
         self.assertRaises(exception.ProcessExecutionError,
diff --git a/nova/utils.py b/nova/utils.py
index 838e11b3c..7880c609c 100644
--- a/nova/utils.py
+++ b/nova/utils.py
@@ -171,10 +171,6 @@ def execute(*cmd, **kwargs):
                                                 stdout=stdout,
                                                 stderr=stderr,
                                                 cmd=' '.join(cmd))
-            # NOTE(termie): this appears to be necessary to let the subprocess
-            #               call clean something up in between calls, without
-            #               it two execute calls in a row hangs the second one
-            greenthread.sleep(0)
             return result
         except ProcessExecutionError:
             if not attempts:
@@ -183,6 +179,11 @@ def execute(*cmd, **kwargs):
                 LOG.debug(_("%r failed. Retrying."), cmd)
                 if delay_on_retry:
                     greenthread.sleep(random.randint(20, 200) / 100.0)
+        finally:
+            # NOTE(termie): this appears to be necessary to let the subprocess
+            #               call clean something up in between calls, without
+            #               it two execute calls in a row hangs the second one
+            greenthread.sleep(0)
 
 
 def ssh_execute(ssh, cmd, process_input=None,