From b13ca667bdd3303bbfcd4e58cc6d773cea09661d Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Wed, 7 Sep 2011 21:48:11 -0700 Subject: catch exceptions from novaclient when talking to child zones. store them and re-raise if no other child zones return any results. If no exceptions are raised but no results are returned, raise a NotFound exception. --- nova/scheduler/api.py | 48 +++++++++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 17 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 55cea5f8f..a5124678d 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -103,22 +103,6 @@ def update_service_capabilities(context, service_name, host, capabilities): return rpc.fanout_cast(context, 'scheduler', kwargs) -def _wrap_method(function, self): - """Wrap method to supply self.""" - def _wrap(*args, **kwargs): - return function(self, *args, **kwargs) - return _wrap - - -def _process(func, zone): - """Worker stub for green thread pool. Give the worker - an authenticated nova client and zone info.""" - nova = novaclient.Client(zone.username, zone.password, None, - zone.api_url) - nova.authenticate() - return func(nova, zone) - - def call_zone_method(context, method_name, errors_to_ignore=None, novaclient_collection_name='zones', zones=None, *args, **kwargs): @@ -166,6 +150,28 @@ def child_zone_helper(zone_list, func): For example, if you are calling server.pause(), the list will be whatever the response from server.pause() is. One entry per child zone called.""" + + def _wrap_method(function, arg1): + """Wrap method to supply an argument.""" + def _wrap(*args, **kwargs): + return function(arg1, *args, **kwargs) + return _wrap + + def _process(func, zone): + """Worker stub for green thread pool. Give the worker + an authenticated nova client and zone info.""" + try: + nova = novaclient.Client(zone.username, zone.password, None, + zone.api_url) + nova.authenticate() + except novaclient_exceptions.BadRequest, e: + url = zone.api_url + LOG.warn(_("Failed request to zone; URL=%(url)s: %(e)s") + % locals()) + return e + else: + return func(nova, zone) + green_pool = greenpool.GreenPool() return [result for result in green_pool.imap( _wrap_method(_process, func), zone_list)] @@ -260,6 +266,8 @@ class reroute_compute(object): if not FLAGS.enable_zone_routing: raise exception.InstanceNotFound(instance_id=item_uuid) + self.item_uuid = item_uuid + zones = db.zone_get_all(context) if not zones: raise exception.InstanceNotFound(instance_id=item_uuid) @@ -342,9 +350,13 @@ class reroute_compute(object): dict {'server':{k:v}}. Others may return a list of them, like {'servers':[{k,v}]}""" reduced_response = [] + found_exception = None for zone_response in zone_responses: if not zone_response: continue + if isinstance(zone_response, BaseException): + found_exception = zone_response + continue server = zone_response.__dict__ @@ -355,7 +367,9 @@ class reroute_compute(object): reduced_response.append(dict(server=server)) if reduced_response: return reduced_response[0] # first for now. - return {} + elif found_exception: + raise found_exception + exception.InstanceNotFound(instance_id=self.item_uuid) def redirect_handler(f): -- cgit From be645283c85d69e2d3cf4f4eabdbb545aaf139bf Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Wed, 7 Sep 2011 23:24:07 -0700 Subject: create a new exception ZoneRequestError to use for returning errors when zone requests couldn't complete --- nova/exception.py | 7 +++++++ nova/scheduler/api.py | 5 ++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/nova/exception.py b/nova/exception.py index 95d8229b5..aa6609461 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -806,3 +806,10 @@ class CannotResizeToSmallerSize(NovaException): class ImageTooLarge(NovaException): message = _("Image is larger than instance type allows") + + +class ZoneRequestError(Error): + def __init__(self, message=None): + if message is None: + message = _("1 or more Zones could not complete the request") + super(ZoneRequestError, self).__init__(message=message) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index a5124678d..05685fc15 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -168,7 +168,10 @@ def child_zone_helper(zone_list, func): url = zone.api_url LOG.warn(_("Failed request to zone; URL=%(url)s: %(e)s") % locals()) - return e + # This is being returned instead of raised, so that when results are + # processed in unmarshal_result() after the greenpool.imap completes, + # the exception can be raised there if no other zones had a response. + return exception.ZoneRequestError() else: return func(nova, zone) -- cgit From d1b1f301583fd67050c45f4c863733f251620a9c Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Wed, 7 Sep 2011 23:39:39 -0700 Subject: typo trying to raise InstanceNotFound when all zones returned nothing --- nova/scheduler/api.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 05685fc15..6081c3f02 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -168,9 +168,10 @@ def child_zone_helper(zone_list, func): url = zone.api_url LOG.warn(_("Failed request to zone; URL=%(url)s: %(e)s") % locals()) - # This is being returned instead of raised, so that when results are - # processed in unmarshal_result() after the greenpool.imap completes, - # the exception can be raised there if no other zones had a response. + # This is being returned instead of raised, so that when + # results are # processed in unmarshal_result() after the + # greenpool.imap completes, # the exception can be raised + # there if no other zones had a response. return exception.ZoneRequestError() else: return func(nova, zone) @@ -372,7 +373,7 @@ class reroute_compute(object): return reduced_response[0] # first for now. elif found_exception: raise found_exception - exception.InstanceNotFound(instance_id=self.item_uuid) + raise exception.InstanceNotFound(instance_id=self.item_uuid) def redirect_handler(f): -- cgit From be7a081976d37b84d93028673d08ab78bc9d8a73 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Wed, 7 Sep 2011 23:45:11 -0700 Subject: comment fix --- nova/scheduler/api.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 6081c3f02..719437b73 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -169,8 +169,8 @@ def child_zone_helper(zone_list, func): LOG.warn(_("Failed request to zone; URL=%(url)s: %(e)s") % locals()) # This is being returned instead of raised, so that when - # results are # processed in unmarshal_result() after the - # greenpool.imap completes, # the exception can be raised + # results are processed in unmarshal_result() after the + # greenpool.imap completes, the exception can be raised # there if no other zones had a response. return exception.ZoneRequestError() else: -- cgit From 973870c82445d4c1ebbd46f2ba7c3817ae5e7f87 Mon Sep 17 00:00:00 2001 From: Chris Behrens Date: Thu, 8 Sep 2011 01:09:22 -0700 Subject: added tests for failure cases talking with zones --- nova/tests/scheduler/test_scheduler.py | 102 +++++++++++++++++++++++++++++++-- 1 file changed, 97 insertions(+), 5 deletions(-) diff --git a/nova/tests/scheduler/test_scheduler.py b/nova/tests/scheduler/test_scheduler.py index a52dd041a..890348192 100644 --- a/nova/tests/scheduler/test_scheduler.py +++ b/nova/tests/scheduler/test_scheduler.py @@ -963,9 +963,14 @@ class FakeZone(object): self.password = password +ZONE_API_URL1 = "http://1.example.com" +ZONE_API_URL2 = "http://2.example.com" + + def zone_get_all(context): return [ - FakeZone(1, 'http://example.com', 'bob', 'xxx'), + FakeZone(1, ZONE_API_URL1, 'bob', 'xxx'), + FakeZone(2, ZONE_API_URL2, 'bob', 'xxx'), ] @@ -1065,7 +1070,9 @@ class ZoneRedirectTest(test.TestCase): def test_unmarshal_single_server(self): decorator = api.reroute_compute("foo") - self.assertEquals(decorator.unmarshall_result([]), {}) + decorator.item_uuid = 'fake_uuid' + self.assertRaises(exception.InstanceNotFound, + decorator.unmarshall_result, []) self.assertEquals(decorator.unmarshall_result( [FakeResource(dict(a=1, b=2)), ]), dict(server=dict(a=1, b=2))) @@ -1079,6 +1086,90 @@ class ZoneRedirectTest(test.TestCase): [FakeResource(dict(_a=1, manager=2)), ]), dict(server={})) + def test_one_zone_down_no_instances(self): + + def _fake_issue_novaclient_command(nova, zone, *args, **kwargs): + return None + + class FakeNovaClientWithFailure(object): + def __init__(self, username, password, method, api_url): + self.api_url = api_url + + def authenticate(self): + if self.api_url == ZONE_API_URL2: + raise novaclient_exceptions.BadRequest('foo') + + self.stubs.Set(api, '_issue_novaclient_command', + _fake_issue_novaclient_command) + self.stubs.Set(api.novaclient, 'Client', FakeNovaClientWithFailure) + + @api.reroute_compute("get") + def do_get(self, context, uuid): + pass + + self.assertRaises(exception.ZoneRequestError, + do_get, None, {}, FAKE_UUID) + + def test_one_zone_down_got_instance(self): + + def _fake_issue_novaclient_command(nova, zone, *args, **kwargs): + class FakeServer(object): + def __init__(self): + self.id = FAKE_UUID + self.test = '1234' + return FakeServer() + + class FakeNovaClientWithFailure(object): + def __init__(self, username, password, method, api_url): + self.api_url = api_url + + def authenticate(self): + if self.api_url == ZONE_API_URL2: + raise novaclient_exceptions.BadRequest('foo') + + self.stubs.Set(api, '_issue_novaclient_command', + _fake_issue_novaclient_command) + self.stubs.Set(api.novaclient, 'Client', FakeNovaClientWithFailure) + + @api.reroute_compute("get") + def do_get(self, context, uuid): + pass + + try: + do_get(None, {}, FAKE_UUID) + except api.RedirectResult, e: + results = e.results + self.assertIn('server', results) + self.assertEqual(results['server']['id'], FAKE_UUID) + self.assertEqual(results['server']['test'], '1234') + except Exception, e: + self.fail(_("RedirectResult should have been raised")) + else: + self.fail(_("RedirectResult should have been raised")) + + def test_zones_up_no_instances(self): + + def _fake_issue_novaclient_command(nova, zone, *args, **kwargs): + return None + + class FakeNovaClientNoFailure(object): + def __init__(self, username, password, method, api_url): + pass + + def authenticate(self): + return + + self.stubs.Set(api, '_issue_novaclient_command', + _fake_issue_novaclient_command) + self.stubs.Set(api.novaclient, 'Client', FakeNovaClientNoFailure) + + @api.reroute_compute("get") + def do_get(self, context, uuid): + pass + + self.assertRaises(exception.InstanceNotFound, + do_get, None, {}, FAKE_UUID) + class FakeServerCollection(object): def get(self, instance_id): @@ -1097,7 +1188,7 @@ class FakeEmptyServerCollection(object): class FakeNovaClient(object): - def __init__(self, collection): + def __init__(self, collection, *args, **kwargs): self.servers = collection @@ -1162,8 +1253,9 @@ class CallZoneMethodTest(test.TestCase): context = {} method = 'do_something' results = api.call_zone_method(context, method) - expected = [(1, 42)] - self.assertEqual(expected, results) + self.assertEqual(len(results), 2) + self.assertIn((1, 42), results) + self.assertIn((2, 42), results) def test_call_zone_method_not_present(self): context = {} -- cgit