diff options
author | Chris Behrens <cbehrens@codestud.com> | 2013-01-14 00:58:10 +0000 |
---|---|---|
committer | Chris Behrens <cbehrens@codestud.com> | 2013-01-15 01:22:32 +0000 |
commit | f6c205f66eef6bac572f9f4176b3bc7f90d57ff0 (patch) | |
tree | b30cd7b63c79411c0bac1562fb3dcee9325bcf12 | |
parent | fe3ab76b82e9242c18a98cb988f213aaec10ddce (diff) | |
download | nova-f6c205f66eef6bac572f9f4176b3bc7f90d57ff0.tar.gz nova-f6c205f66eef6bac572f9f4176b3bc7f90d57ff0.tar.xz nova-f6c205f66eef6bac572f9f4176b3bc7f90d57ff0.zip |
More HostAPI() cleanup for cells.
This patch contains further changes needed to support HostAPI() with
cells. This moves the formatting of hosts API extension responses back
to the extension itself. Methods for getting service entries have been
added to HostAPI() that can be overriden in a subclass for cells.
Also contains a performance improvement for 'show' in hosts extension.
Change-Id: Ifd8b68ff8d7495f537dff5e01bd3df75c6c5930a
-rw-r--r-- | nova/api/openstack/compute/contrib/hosts.py | 78 | ||||
-rw-r--r-- | nova/compute/api.py | 157 | ||||
-rw-r--r-- | nova/db/sqlalchemy/api.py | 9 | ||||
-rw-r--r-- | nova/tests/api/openstack/compute/contrib/test_hosts.py | 10 | ||||
-rw-r--r-- | nova/tests/compute/test_compute.py | 75 | ||||
-rw-r--r-- | nova/tests/compute/test_host_api.py | 175 |
6 files changed, 224 insertions, 280 deletions
diff --git a/nova/api/openstack/compute/contrib/hosts.py b/nova/api/openstack/compute/contrib/hosts.py index 52487c305..d1b39d6db 100644 --- a/nova/api/openstack/compute/contrib/hosts.py +++ b/nova/api/openstack/compute/contrib/hosts.py @@ -124,10 +124,17 @@ class HostController(object): """ context = req.environ['nova.context'] authorize(context) + filters = {} zone = req.GET.get('zone', None) - data = self.api.list_hosts(context, zone) - - return {'hosts': data} + if zone: + filters['availability_zone'] = zone + services = self.api.service_get_all(context, filters=filters) + hosts = [] + for service in services: + hosts.append({'host_name': service['host'], + 'service': service['topic'], + 'zone': service['availability_zone']}) + return {'hosts': hosts} @wsgi.serializers(xml=HostUpdateTemplate) @wsgi.deserializers(xml=HostUpdateDeserializer) @@ -243,6 +250,55 @@ class HostController(object): def reboot(self, req, id): return self._host_power_action(req, host_name=id, action="reboot") + @staticmethod + def _get_total_resources(host_name, compute_node): + return {'resource': {'host': host_name, + 'project': '(total)', + 'cpu': compute_node['vcpus'], + 'memory_mb': compute_node['memory_mb'], + 'disk_gb': compute_node['local_gb']}} + + @staticmethod + def _get_used_now_resources(host_name, compute_node): + return {'resource': {'host': host_name, + 'project': '(used_now)', + 'cpu': compute_node['vcpus_used'], + 'memory_mb': compute_node['memory_mb_used'], + 'disk_gb': compute_node['local_gb_used']}} + + @staticmethod + def _get_resource_totals_from_instances(host_name, instances): + cpu_sum = 0 + mem_sum = 0 + hdd_sum = 0 + for instance in instances: + cpu_sum += instance['vcpus'] + mem_sum += instance['memory_mb'] + hdd_sum += instance['root_gb'] + instance['ephemeral_gb'] + + return {'resource': {'host': host_name, + 'project': '(used_max)', + 'cpu': cpu_sum, + 'memory_mb': mem_sum, + 'disk_gb': hdd_sum}} + + @staticmethod + def _get_resources_by_project(host_name, instances): + # Getting usage resource per project + project_map = {} + for instance in instances: + resource = project_map.setdefault(instance['project_id'], + {'host': host_name, + 'project': instance['project_id'], + 'cpu': 0, + 'memory_mb': 0, + 'disk_gb': 0}) + resource['cpu'] += instance['vcpus'] + resource['memory_mb'] += instance['memory_mb'] + resource['disk_gb'] += (instance['root_gb'] + + instance['ephemeral_gb']) + return project_map + @wsgi.serializers(xml=HostShowTemplate) def show(self, req, id): """Shows the physical/usage resource given by hosts. @@ -256,14 +312,26 @@ class HostController(object): 'cpu': 1, 'memory_mb': 2048, 'disk_gb': 30} """ context = req.environ['nova.context'] + host_name = id try: - data = self.api.describe_host(context, id) + service = self.api.service_get_by_compute_host(context, host_name) except exception.NotFound as e: raise webob.exc.HTTPNotFound(explanation=e.message) except exception.AdminRequired: msg = _("Describe-resource is admin only functionality") raise webob.exc.HTTPForbidden(explanation=msg) - return {'host': data} + compute_node = service['compute_node'][0] + instances = self.api.instance_get_all_by_host(context, host_name) + resources = [self._get_total_resources(host_name, compute_node)] + resources.append(self._get_used_now_resources(host_name, + compute_node)) + resources.append(self._get_resource_totals_from_instances(host_name, + instances)) + by_proj_resources = self._get_resources_by_project(host_name, + instances) + for resource in by_proj_resources.itervalues(): + resources.append({'resource': resource}) + return {'host': resources} class Hosts(extensions.ExtensionDescriptor): diff --git a/nova/compute/api.py b/nova/compute/api.py index bfb5c6195..9ccd35573 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -2187,139 +2187,76 @@ class API(base.Base): disk_over_commit, instance, host_name) -def check_host(fn): - """Decorator that makes sure that the host exists.""" - def wrapped(self, context, host_name, *args, **kwargs): - if self.does_host_exist(context, host_name): - return fn(self, context, host_name, *args, **kwargs) - else: - raise exception.HostNotFound(host=host_name) - return wrapped - - class HostAPI(base.Base): """Sub-set of the Compute Manager API for managing host operations.""" - def __init__(self): - self.compute_rpcapi = compute_rpcapi.ComputeAPI() + def __init__(self, rpcapi=None): + self.rpcapi = rpcapi or compute_rpcapi.ComputeAPI() super(HostAPI, self).__init__() - @check_host + def _assert_host_exists(self, context, host_name): + """Raise HostNotFound if compute host doesn't exist.""" + if not self.db.service_get_by_host_and_topic(context, host_name, + CONF.compute_topic): + raise exception.HostNotFound(host=host_name) + def set_host_enabled(self, context, host_name, enabled): """Sets the specified host's ability to accept new instances.""" # NOTE(comstud): No instance_uuid argument to this compute manager # call - return self.compute_rpcapi.set_host_enabled(context, enabled=enabled, + self._assert_host_exists(context, host_name) + return self.rpcapi.set_host_enabled(context, enabled=enabled, host=host_name) - @check_host def get_host_uptime(self, context, host_name): """Returns the result of calling "uptime" on the target host.""" # NOTE(comstud): No instance_uuid argument to this compute manager # call - return self.compute_rpcapi.get_host_uptime(context, host=host_name) + self._assert_host_exists(context, host_name) + return self.rpcapi.get_host_uptime(context, host=host_name) - @check_host def host_power_action(self, context, host_name, action): """Reboots, shuts down or powers up the host.""" - return self.compute_rpcapi.host_power_action(context, action=action, + self._assert_host_exists(context, host_name) + return self.rpcapi.host_power_action(context, action=action, host=host_name) - def list_hosts(self, context, zone=None, service=None): - """Returns a summary list of enabled hosts, optionally filtering - by zone and/or service type. + def set_host_maintenance(self, context, host_name, mode): + """Start/Stop host maintenance window. On start, it triggers + guest VMs evacuation.""" + self._assert_host_exists(context, host_name) + return self.rpcapi.host_maintenance_mode(context, + host_param=host_name, mode=mode, host=host_name) + + def service_get_all(self, context, filters=None): + """Returns a list of services, optionally filtering the results. + + If specified, 'filters' should be a dictionary containing services + attributes and matching values. Ie, to get a list of services for + the 'compute' topic, use filters={'topic': 'compute'}. """ - LOG.debug(_("Listing hosts")) + if filters is None: + filters = {} services = self.db.service_get_all(context, False) - services = availability_zones.set_availability_zones(context, services) - if zone: - services = [s for s in services if s['availability_zone'] == zone] - hosts = [] - for host in services: - hosts.append({'host_name': host['host'], 'service': host['topic'], - 'zone': host['availability_zone']}) - if service: - hosts = [host for host in hosts - if host["service"] == service] - return hosts - - def does_host_exist(self, context, host_name): - """ - Returns True if the host with host_name exists, False otherwise - """ - return self.db.service_does_host_exist(context, host_name) + services = availability_zones.set_availability_zones(context, + services) + ret_services = [] + for service in services: + for key, val in filters.iteritems(): + if service[key] != val: + break + else: + # All filters matched. + ret_services.append(service) + return ret_services - def describe_host(self, context, host_name): - """ - Returns information about a host in this kind of format: - :returns: - ex.:: - {'host': 'hostname', - 'project': 'admin', - 'cpu': 1, - 'memory_mb': 2048, - 'disk_gb': 30} - """ - # Getting compute node info and related instances info - try: - compute_ref = self.db.service_get_by_compute_host(context, - host_name) - except exception.ComputeHostNotFound: - raise exception.HostNotFound(host=host_name) - instance_refs = self.db.instance_get_all_by_host(context, - compute_ref['host']) - - # Getting total available/used resource - compute_ref = compute_ref['compute_node'][0] - resources = [{'resource': {'host': host_name, 'project': '(total)', - 'cpu': compute_ref['vcpus'], - 'memory_mb': compute_ref['memory_mb'], - 'disk_gb': compute_ref['local_gb']}}, - {'resource': {'host': host_name, 'project': '(used_now)', - 'cpu': compute_ref['vcpus_used'], - 'memory_mb': compute_ref['memory_mb_used'], - 'disk_gb': compute_ref['local_gb_used']}}] - - cpu_sum = 0 - mem_sum = 0 - hdd_sum = 0 - for i in instance_refs: - cpu_sum += i['vcpus'] - mem_sum += i['memory_mb'] - hdd_sum += i['root_gb'] + i['ephemeral_gb'] - - resources.append({'resource': {'host': host_name, - 'project': '(used_max)', - 'cpu': cpu_sum, - 'memory_mb': mem_sum, - 'disk_gb': hdd_sum}}) - - # Getting usage resource per project - project_ids = [i['project_id'] for i in instance_refs] - project_ids = list(set(project_ids)) - for project_id in project_ids: - vcpus = [i['vcpus'] for i in instance_refs - if i['project_id'] == project_id] - - mem = [i['memory_mb'] for i in instance_refs - if i['project_id'] == project_id] - - disk = [i['root_gb'] + i['ephemeral_gb'] for i in instance_refs - if i['project_id'] == project_id] - - resources.append({'resource': {'host': host_name, - 'project': project_id, - 'cpu': sum(vcpus), - 'memory_mb': sum(mem), - 'disk_gb': sum(disk)}}) - return resources - - @check_host - def set_host_maintenance(self, context, host, mode): - """Start/Stop host maintenance window. On start, it triggers - guest VMs evacuation.""" - return self.compute_rpcapi.host_maintenance_mode(context, - host_param=host, mode=mode, host=host) + def service_get_by_compute_host(self, context, host_name): + """Get service entry for the given compute hostname.""" + return self.db.service_get_by_compute_host(context, host_name) + + def instance_get_all_by_host(self, context, host_name): + """Return all instances on the given host.""" + return self.db.instance_get_all_by_host(context, host_name) class AggregateAPI(base.Base): diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index e51d7b685..5cac0b065 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -337,15 +337,6 @@ def service_get_all(context, disabled=None): @require_admin_context -def service_does_host_exist(context, host_name, include_disabled): - query = get_session().query(func.count(models.Service.host)).\ - filter_by(host=host_name) - if not include_disabled: - query = query.filter_by(disabled=False) - return query.scalar() > 0 - - -@require_admin_context def service_get_all_by_topic(context, topic): return model_query(context, models.Service, read_deleted="no").\ filter_by(disabled=False).\ diff --git a/nova/tests/api/openstack/compute/contrib/test_hosts.py b/nova/tests/api/openstack/compute/contrib/test_hosts.py index be4465cf9..e103b5b19 100644 --- a/nova/tests/api/openstack/compute/contrib/test_hosts.py +++ b/nova/tests/api/openstack/compute/contrib/test_hosts.py @@ -32,8 +32,10 @@ def stub_service_get_all(context, disabled=None): return fake_hosts.SERVICES_LIST -def stub_service_does_host_exist(context, host_name): - return host_name in [row['host'] for row in stub_service_get_all(context)] +def stub_service_get_by_host_and_topic(context, host_name, topic): + for service in stub_service_get_all(context): + if service['host'] == host_name and service['topic'] == topic: + return service def stub_set_host_enabled(context, host_name, enabled): @@ -130,8 +132,8 @@ class HostTestCase(test.TestCase): self.stubs.Set(db, 'service_get_all', stub_service_get_all) # Only hosts in our fake DB exist - self.stubs.Set(db, 'service_does_host_exist', - stub_service_does_host_exist) + self.stubs.Set(db, 'service_get_by_host_and_topic', + stub_service_get_by_host_and_topic) # 'host_c1' always succeeds, and 'host_c2' self.stubs.Set(self.hosts_api, 'set_host_enabled', stub_set_host_enabled) diff --git a/nova/tests/compute/test_compute.py b/nova/tests/compute/test_compute.py index 08d9451b3..9dedd782b 100644 --- a/nova/tests/compute/test_compute.py +++ b/nova/tests/compute/test_compute.py @@ -6090,81 +6090,6 @@ class ComputePolicyTestCase(BaseTestCase): availability_zone='1:1') -class ComputeHostAPITestCase(BaseTestCase): - def setUp(self): - super(ComputeHostAPITestCase, self).setUp() - self.host_api = compute_api.HostAPI() - - def _rpc_call_stub(self, call_info): - def fake_rpc_call(context, topic, msg, timeout=None): - call_info['context'] = context - call_info['topic'] = topic - call_info['msg'] = msg - self.stubs.Set(rpc, 'call', fake_rpc_call) - - def _pretend_fake_host_exists(self, ctxt): - """Sets it so that the host API always thinks that 'fake_host' - exists""" - self.mox.StubOutWithMock(self.host_api, 'does_host_exist') - self.host_api.does_host_exist(ctxt, 'fake_host').AndReturn(True) - self.mox.ReplayAll() - - def test_set_host_enabled(self): - ctxt = context.get_admin_context() - call_info = {} - self._rpc_call_stub(call_info) - - self._pretend_fake_host_exists(ctxt) - self.host_api.set_host_enabled(ctxt, 'fake_host', 'fake_enabled') - self.assertEqual(call_info['context'], ctxt) - self.assertEqual(call_info['topic'], 'compute.fake_host') - self.assertEqual(call_info['msg'], - {'method': 'set_host_enabled', - 'args': {'enabled': 'fake_enabled'}, - 'version': compute_rpcapi.ComputeAPI.BASE_RPC_API_VERSION}) - - def test_get_host_uptime(self): - ctxt = context.RequestContext('fake', 'fake') - call_info = {} - self._rpc_call_stub(call_info) - - self._pretend_fake_host_exists(ctxt) - self.host_api.get_host_uptime(ctxt, 'fake_host') - self.assertEqual(call_info['context'], ctxt) - self.assertEqual(call_info['topic'], 'compute.fake_host') - self.assertEqual(call_info['msg'], - {'method': 'get_host_uptime', - 'args': {}, - 'version': compute_rpcapi.ComputeAPI.BASE_RPC_API_VERSION}) - - def test_host_power_action(self): - ctxt = context.get_admin_context() - call_info = {} - self._rpc_call_stub(call_info) - self._pretend_fake_host_exists(ctxt) - self.host_api.host_power_action(ctxt, 'fake_host', 'fake_action') - self.assertEqual(call_info['context'], ctxt) - self.assertEqual(call_info['topic'], 'compute.fake_host') - self.assertEqual(call_info['msg'], - {'method': 'host_power_action', - 'args': {'action': 'fake_action'}, - 'version': - compute_rpcapi.ComputeAPI.BASE_RPC_API_VERSION}) - - def test_set_host_maintenance(self): - ctxt = context.get_admin_context() - call_info = {} - self._rpc_call_stub(call_info) - self._pretend_fake_host_exists(ctxt) - self.host_api.set_host_maintenance(ctxt, 'fake_host', 'fake_mode') - self.assertEqual(call_info['context'], ctxt) - self.assertEqual(call_info['topic'], 'compute.fake_host') - self.assertEqual(call_info['msg'], - {'method': 'host_maintenance_mode', - 'args': {'host': 'fake_host', 'mode': 'fake_mode'}, - 'version': compute_rpcapi.ComputeAPI.BASE_RPC_API_VERSION}) - - class KeypairAPITestCase(BaseTestCase): def setUp(self): super(KeypairAPITestCase, self).setUp() diff --git a/nova/tests/compute/test_host_api.py b/nova/tests/compute/test_host_api.py index 0af1d6766..95d3c4926 100644 --- a/nova/tests/compute/test_host_api.py +++ b/nova/tests/compute/test_host_api.py @@ -13,93 +13,114 @@ # License for the specific language governing permissions and limitations # under the License. -from nova.compute import api +from nova import compute +from nova.compute import rpcapi as compute_rpcapi from nova import context -from nova import db -from nova import exception +from nova.openstack.common import rpc from nova import test -from nova.tests import fake_hosts -class HostApiTestCase(test.TestCase): - """ - Tests 'host' subset of the compute api - """ - +class ComputeHostAPITestCase(test.TestCase): def setUp(self): - super(HostApiTestCase, self).setUp() - self.compute_rpcapi = api.compute_rpcapi - self.api = api.HostAPI() + super(ComputeHostAPITestCase, self).setUp() + self.host_api = compute.HostAPI() + self.ctxt = context.get_admin_context() - def test_bad_host_set_enabled(self): - """ - Tests that actions on single hosts that don't exist blow up without - having to reach the host via rpc. Should raise HostNotFound if you - try to update a host that is not in the DB + def _mock_rpc_call(self, expected_message, result=None): + if result is None: + result = 'fake-result' + self.mox.StubOutWithMock(rpc, 'call') + rpc.call(self.ctxt, 'compute.fake_host', + expected_message, None).AndReturn(result) + + def _mock_assert_host_exists(self): + """Sets it so that the host API always thinks that 'fake_host' + exists. """ - self.assertRaises(exception.HostNotFound, self.api.set_host_enabled, - context.get_admin_context(), "bogus_host_name", False) + self.mox.StubOutWithMock(self.host_api, '_assert_host_exists') + self.host_api._assert_host_exists(self.ctxt, 'fake_host') + + def test_set_host_enabled(self): + self._mock_assert_host_exists() + self._mock_rpc_call( + {'method': 'set_host_enabled', + 'args': {'enabled': 'fake_enabled'}, + 'version': compute_rpcapi.ComputeAPI.BASE_RPC_API_VERSION}) + + self.mox.ReplayAll() + result = self.host_api.set_host_enabled(self.ctxt, 'fake_host', + 'fake_enabled') + self.assertEqual('fake-result', result) + + def test_get_host_uptime(self): + self._mock_assert_host_exists() + self._mock_rpc_call( + {'method': 'get_host_uptime', + 'args': {}, + 'version': compute_rpcapi.ComputeAPI.BASE_RPC_API_VERSION}) + self.mox.ReplayAll() + result = self.host_api.get_host_uptime(self.ctxt, 'fake_host') + self.assertEqual('fake-result', result) + + def test_host_power_action(self): + self._mock_assert_host_exists() + self._mock_rpc_call( + {'method': 'host_power_action', + 'args': {'action': 'fake_action'}, + 'version': compute_rpcapi.ComputeAPI.BASE_RPC_API_VERSION}) + self.mox.ReplayAll() + result = self.host_api.host_power_action(self.ctxt, 'fake_host', + 'fake_action') + self.assertEqual('fake-result', result) - def test_list_compute_hosts(self): - ctx = context.get_admin_context() - self.mox.StubOutWithMock(db, 'service_get_all') - db.service_get_all(ctx, False).AndReturn(fake_hosts.SERVICES_LIST) + def test_set_host_maintenance(self): + self._mock_assert_host_exists() + self._mock_rpc_call( + {'method': 'host_maintenance_mode', + 'args': {'host': 'fake_host', 'mode': 'fake_mode'}, + 'version': compute_rpcapi.ComputeAPI.BASE_RPC_API_VERSION}) self.mox.ReplayAll() - compute_hosts = self.api.list_hosts(ctx, service="compute") + result = self.host_api.set_host_maintenance(self.ctxt, 'fake_host', + 'fake_mode') + self.assertEqual('fake-result', result) + + def test_service_get_all(self): + services = [dict(id=1, key1='val1', key2='val2', topic='compute', + host='host1'), + dict(id=2, key1='val2', key3='val3', topic='compute', + host='host2')] + exp_services = [] + for service in services: + exp_service = {} + exp_service.update(availability_zone='nova', **service) + exp_services.append(exp_service) + + self.mox.StubOutWithMock(self.host_api.db, + 'service_get_all') + + # Test no filters + self.host_api.db.service_get_all(self.ctxt, False).AndReturn( + services) + self.mox.ReplayAll() + result = self.host_api.service_get_all(self.ctxt) self.mox.VerifyAll() - expected = [host for host in fake_hosts.HOST_LIST - if host["service"] == "compute"] - self.assertEqual(expected, compute_hosts) + self.assertEqual(exp_services, result) - def test_describe_host(self): - """ - Makes sure that describe_host returns the correct information - given our fake input. - """ - ctx = context.get_admin_context() - self.mox.StubOutWithMock(db, 'service_get_by_compute_host') - host_name = 'host_c1' - db.service_get_by_compute_host(ctx, host_name).AndReturn( - {'host': 'fake_host', - 'compute_node': [ - {'vcpus': 4, - 'vcpus_used': 1, - 'memory_mb': 8192, - 'memory_mb_used': 2048, - 'local_gb': 1024, - 'local_gb_used': 648} - ] - }) - self.mox.StubOutWithMock(db, 'instance_get_all_by_host') - db.instance_get_all_by_host(ctx, 'fake_host').AndReturn( - [{'project_id': 42, - 'vcpus': 1, - 'memory_mb': 2048, - 'root_gb': 648, - 'ephemeral_gb': 0, - }]) + # Test no filters #2 + self.mox.ResetAll() + self.host_api.db.service_get_all(self.ctxt, False).AndReturn( + services) + self.mox.ReplayAll() + result = self.host_api.service_get_all(self.ctxt, filters={}) + self.mox.VerifyAll() + self.assertEqual(exp_services, result) + + # Test w/ filter + self.mox.ResetAll() + self.host_api.db.service_get_all(self.ctxt, False).AndReturn( + services) self.mox.ReplayAll() - result = self.api.describe_host(ctx, host_name) - self.assertEqual(result, - [{'resource': {'cpu': 4, - 'disk_gb': 1024, - 'host': 'host_c1', - 'memory_mb': 8192, - 'project': '(total)'}}, - {'resource': {'cpu': 1, - 'disk_gb': 648, - 'host': 'host_c1', - 'memory_mb': 2048, - 'project': '(used_now)'}}, - {'resource': {'cpu': 1, - 'disk_gb': 648, - 'host': 'host_c1', - 'memory_mb': 2048, - 'project': '(used_max)'}}, - {'resource': {'cpu': 1, - 'disk_gb': 648, - 'host': 'host_c1', - 'memory_mb': 2048, - 'project': 42}}] - ) + result = self.host_api.service_get_all(self.ctxt, + filters=dict(key1='val2')) self.mox.VerifyAll() + self.assertEqual([exp_services[1]], result) |