From 655eb8fbd21376e694f8134e42f10ddbc1aafb0e Mon Sep 17 00:00:00 2001 From: John Tran Date: Wed, 6 Apr 2011 18:22:03 -0700 Subject: ec2 api run_instances checks for image status must be 'available'. Overhauled test_run_instances for working set of test assertions --- nova/api/ec2/cloud.py | 10 +++++++- nova/tests/test_cloud.py | 62 +++++++++++++++++++++++++----------------------- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 58effd134..0ea0e3603 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -814,10 +814,18 @@ class CloudController(object): if kwargs.get('ramdisk_id'): ramdisk = self._get_image(context, kwargs['ramdisk_id']) kwargs['ramdisk_id'] = ramdisk['id'] + image = self._get_image(context, kwargs['image_id']) + if not image: + raise exception.NotFound(_('Image %s not found') % + kwargs['image_id']) + if not 'properties' in image or \ + (not 'image_state' in image['properties']) or \ + (image['properties']['image_state'] is not 'available'): + raise exception.ApiError(_('Image must be available')) instances = self.compute_api.create(context, instance_type=instance_types.get_by_type( kwargs.get('instance_type', None)), - image_id=self._get_image(context, kwargs['image_id'])['id'], + image_id = image['id'], min_count=int(kwargs.get('min_count', max_count)), max_count=max_count, kernel_id=kwargs.get('kernel_id'), diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 5cb969979..85f3a8e87 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -36,12 +36,12 @@ from nova import rpc from nova import service from nova import test from nova import utils +from nova import exception from nova.auth import manager from nova.compute import power_state from nova.api.ec2 import cloud from nova.api.ec2 import ec2utils from nova.image import local -from nova.exception import NotFound FLAGS = flags.FLAGS @@ -226,7 +226,7 @@ class CloudTestCase(test.TestCase): 'type': 'machine'}}] def fake_show_none(meh, context, id): - raise NotFound + raise exception.NotFound self.stubs.Set(local.LocalImageService, 'detail', fake_detail) # list all @@ -244,7 +244,7 @@ class CloudTestCase(test.TestCase): self.stubs.UnsetAll() self.stubs.Set(local.LocalImageService, 'show', fake_show_none) self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show_none) - self.assertRaises(NotFound, describe_images, + self.assertRaises(exception.NotFound, describe_images, self.context, ['ami-fake']) def test_console_output(self): @@ -307,39 +307,41 @@ class CloudTestCase(test.TestCase): self.cloud.delete_key_pair(self.context, 'test') def test_run_instances(self): - if FLAGS.connection_type == 'fake': - LOG.debug(_("Can't test instances without a real virtual env.")) - return + allinst = db.instance_get_all(context.get_admin_context()) + self.assertEqual(0, len(allinst)) + def fake_show_decrypt(meh, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, + 'type': 'machine', 'image_state': 'decrypting'}} + + def fake_show_avail(meh, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, + 'type': 'machine', 'image_state': 'available'}} + image_id = FLAGS.default_image instance_type = FLAGS.default_instance_type max_count = 1 kwargs = {'image_id': image_id, 'instance_type': instance_type, 'max_count': max_count} - rv = self.cloud.run_instances(self.context, **kwargs) - # TODO: check for proper response - instance_id = rv['reservationSet'][0].keys()[0] - instance = rv['reservationSet'][0][instance_id][0] - LOG.debug(_("Need to watch instance %s until it's running..."), - instance['instance_id']) - while True: - greenthread.sleep(1) - info = self.cloud._get_instance(instance['instance_id']) - LOG.debug(info['state']) - if info['state'] == power_state.RUNNING: - break - self.assert_(rv) - - if FLAGS.connection_type != 'fake': - time.sleep(45) # Should use boto for polling here - for reservations in rv['reservationSet']: - # for res_id in reservations.keys(): - # LOG.debug(reservations[res_id]) - # for instance in reservations[res_id]: - for instance in reservations[reservations.keys()[0]]: - instance_id = instance['instance_id'] - LOG.debug(_("Terminating instance %s"), instance_id) - rv = self.compute.terminate_instance(instance_id) + run_instances = self.cloud.run_instances + # when image doesn't have 'image_state' attr at all + self.assertRaises(exception.ApiError, run_instances, + self.context, **kwargs) + # when image has 'image_state' yet not 'available' + self.stubs.UnsetAll() + self.stubs.Set(local.LocalImageService, 'show', fake_show_decrypt) + self.assertRaises(exception.ApiError, run_instances, + self.context, **kwargs) + # when image has valid image_state + self.stubs.UnsetAll() + self.stubs.Set(local.LocalImageService, 'show', fake_show_avail) + result = run_instances(self.context, **kwargs) + instance = result['instancesSet'][0] + self.assertEqual(instance['imageId'], 'ami-00000001') + self.assertEqual(instance['displayName'], 'Server 1') + self.assertEqual(instance['instanceId'], 'i-00000001') + self.assertEqual(instance['instanceState']['name'], 'scheduling') + self.assertEqual(instance['instanceType'], 'm1.small') def test_update_of_instance_display_fields(self): inst = db.instance_create(self.context, {}) -- cgit From 10db492376a8bb8409e3fb3c33707865ac0f3ee7 Mon Sep 17 00:00:00 2001 From: John Tran Date: Mon, 2 May 2011 14:25:21 -0700 Subject: implemented review suggestion EAFP style, and fixed test stub fake_show needs to have image_state = available or other tests will fail --- nova/api/ec2/cloud.py | 14 +++++++++----- nova/tests/test_cloud.py | 35 ++++++++++++++++++----------------- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 0ea0e3603..5dc608139 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -815,17 +815,21 @@ class CloudController(object): ramdisk = self._get_image(context, kwargs['ramdisk_id']) kwargs['ramdisk_id'] = ramdisk['id'] image = self._get_image(context, kwargs['image_id']) - if not image: + if not image: raise exception.NotFound(_('Image %s not found') % kwargs['image_id']) - if not 'properties' in image or \ - (not 'image_state' in image['properties']) or \ - (image['properties']['image_state'] is not 'available'): + try: + available = (image['properties']['image_state'] == 'available') + except KeyError: + available = False + + if not available: raise exception.ApiError(_('Image must be available')) + instances = self.compute_api.create(context, instance_type=instance_types.get_by_type( kwargs.get('instance_type', None)), - image_id = image['id'], + image_id=image['id'], min_count=int(kwargs.get('min_count', max_count)), max_count=max_count, kernel_id=kwargs.get('kernel_id'), diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 85f3a8e87..da2fce06b 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -30,13 +30,13 @@ from eventlet import greenthread from nova import context from nova import crypto from nova import db +from nova import exception from nova import flags from nova import log as logging from nova import rpc from nova import service from nova import test from nova import utils -from nova import exception from nova.auth import manager from nova.compute import power_state from nova.api.ec2 import cloud @@ -73,7 +73,7 @@ class CloudTestCase(test.TestCase): def fake_show(meh, context, id): return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, - 'type': 'machine'}} + 'type': 'machine', 'image_state': 'available'}} self.stubs.Set(local.LocalImageService, 'show', fake_show) self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show) @@ -307,15 +307,16 @@ class CloudTestCase(test.TestCase): self.cloud.delete_key_pair(self.context, 'test') def test_run_instances(self): - allinst = db.instance_get_all(context.get_admin_context()) - self.assertEqual(0, len(allinst)) - def fake_show_decrypt(meh, context, id): + all_instances = db.instance_get_all(context.get_admin_context()) + self.assertEqual(0, len(all_instances)) + + def fake_show_decrypt(self, context, id): return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, 'type': 'machine', 'image_state': 'decrypting'}} - def fake_show_avail(meh, context, id): + def fake_show_no_state(self, context, id): return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, - 'type': 'machine', 'image_state': 'available'}} + 'type': 'machine'}} image_id = FLAGS.default_image instance_type = FLAGS.default_instance_type @@ -324,17 +325,7 @@ class CloudTestCase(test.TestCase): 'instance_type': instance_type, 'max_count': max_count} run_instances = self.cloud.run_instances - # when image doesn't have 'image_state' attr at all - self.assertRaises(exception.ApiError, run_instances, - self.context, **kwargs) - # when image has 'image_state' yet not 'available' - self.stubs.UnsetAll() - self.stubs.Set(local.LocalImageService, 'show', fake_show_decrypt) - self.assertRaises(exception.ApiError, run_instances, - self.context, **kwargs) # when image has valid image_state - self.stubs.UnsetAll() - self.stubs.Set(local.LocalImageService, 'show', fake_show_avail) result = run_instances(self.context, **kwargs) instance = result['instancesSet'][0] self.assertEqual(instance['imageId'], 'ami-00000001') @@ -342,6 +333,16 @@ class CloudTestCase(test.TestCase): self.assertEqual(instance['instanceId'], 'i-00000001') self.assertEqual(instance['instanceState']['name'], 'scheduling') self.assertEqual(instance['instanceType'], 'm1.small') + # when image doesn't have 'image_state' attr at all + self.stubs.UnsetAll() + self.stubs.Set(local.LocalImageService, 'show', fake_show_no_state) + self.assertRaises(exception.ApiError, run_instances, + self.context, **kwargs) + # when image has 'image_state' yet not 'available' + self.stubs.UnsetAll() + self.stubs.Set(local.LocalImageService, 'show', fake_show_decrypt) + self.assertRaises(exception.ApiError, run_instances, + self.context, **kwargs) def test_update_of_instance_display_fields(self): inst = db.instance_create(self.context, {}) -- cgit From 0a3da155228228d3f0eeac1efdea1e29eef2f3a0 Mon Sep 17 00:00:00 2001 From: John Tran Date: Thu, 12 May 2011 12:04:39 -0700 Subject: changed NotFound exception to ImageNotFound --- nova/api/ec2/cloud.py | 3 +-- nova/tests/test_cloud.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 8a54d23f2..ad8c3fe90 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -823,8 +823,7 @@ class CloudController(object): kwargs['ramdisk_id'] = ramdisk['id'] image = self._get_image(context, kwargs['image_id']) if not image: - raise exception.NotFound(_('Image %s not found') % - kwargs['image_id']) + raise exception.ImageNotFound(kwargs['image_id']) try: available = (image['properties']['image_state'] == 'available') except KeyError: diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 202dc36bc..ebfb5ee44 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -227,7 +227,7 @@ class CloudTestCase(test.TestCase): 'type': 'machine'}}] def fake_show_none(meh, context, id): - raise exception.NotFound + raise exception.ImageNotFound self.stubs.Set(local.LocalImageService, 'detail', fake_detail) # list all @@ -245,7 +245,7 @@ class CloudTestCase(test.TestCase): self.stubs.UnsetAll() self.stubs.Set(local.LocalImageService, 'show', fake_show_none) self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show_none) - self.assertRaises(exception.NotFound, describe_images, + self.assertRaises(exception.ImageNotFound, describe_images, self.context, ['ami-fake']) def test_describe_image_attribute(self): -- cgit From a4ea9ac61568ce5f8300a5ba138f0ac10c79b43c Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Mon, 16 May 2011 15:59:01 -0700 Subject: fix for lp783705 - remove nwfilters when instance is terminated --- nova/tests/test_virt.py | 42 ++++++++++++++++++++++++++++++++++++++++++ nova/virt/libvirt_conn.py | 27 +++++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 1311ba361..babb5de9b 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -1045,3 +1045,45 @@ class NWFilterTestCase(test.TestCase): network_info, "fake") self.assertEquals(len(result), 3) + + def test_unfilter_instance_undefines_nwfilters(self): + class FakeNWFilter: + def __init__(self): + self.undefine_call_count = 0 + + def undefine(self): + self.undefine_call_count += 1 + pass + + fakefilter = FakeNWFilter() + + def _nwfilterLookupByName(ignore): + return fakefilter + + def _filterDefineXMLMock(xml): + return True + + admin_ctxt = context.get_admin_context() + + self.fw._conn.nwfilterDefineXML = _filterDefineXMLMock + self.fw._conn.nwfilterLookupByName = _nwfilterLookupByName + + instance_ref = self._create_instance() + inst_id = instance_ref['id'] + instance = db.instance_get(self.context, inst_id) + + ip = '10.11.12.13' + network_ref = db.project_get_network(self.context, 'fake') + fixed_ip = {'address': ip, 'network_id': network_ref['id']} + db.fixed_ip_create(admin_ctxt, fixed_ip) + db.fixed_ip_update(admin_ctxt, ip, {'allocated': True, + 'instance_id': inst_id}) + self.fw.setup_basic_filtering(instance) + self.fw.prepare_instance_filter(instance) + self.fw.apply_instance_filter(instance) + self.fw.unfilter_instance(instance) + + # should attempt to undefine 2 filters: instance and instance-secgroup + self.assertEquals(fakefilter.undefine_call_count, 2) + + db.instance_destroy(admin_ctxt, instance_ref['id']) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 555e44ce2..706973176 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1835,8 +1835,30 @@ class NWFilterFirewall(FirewallDriver): tpool.execute(self._conn.nwfilterDefineXML, xml) def unfilter_instance(self, instance): - # Nothing to do - pass + """Clear out the nwfilter rules.""" + network_info = _get_network_info(instance) + instance_name = instance.name + for (network, mapping) in network_info: + nic_id = mapping['mac'].replace(':', '') + instance_filter_name = self._instance_filter_name(instance, nic_id) + + try: + self._conn.nwfilterLookupByName(instance_filter_name).\ + undefine() + except libvirt.libvirtError: + LOG.debug(_('The nwfilter(%(instance_filter_name)s) for ' + '%(instance_name)s is not found.') % locals()) + + instance_secgroup_filter_name =\ + '%s-secgroup' % (self._instance_filter_name(instance)) + + try: + self._conn.nwfilterLookupByName(instance_secgroup_filter_name).\ + undefine() + except libvirt.libvirtError: + # This will happen if called by IptablesFirewallDriver + LOG.debug(_('The nwfilter(%(instance_secgroup_filter_name)s) for ' + '%(instance_name)s is not found.') % locals()) def prepare_instance_filter(self, instance, network_info=None): """ @@ -2000,6 +2022,7 @@ class IptablesFirewallDriver(FirewallDriver): if self.instances.pop(instance['id'], None): self.remove_filters_for_instance(instance) self.iptables.apply() + self.nwfilter.unfilter_instance(instance) else: LOG.info(_('Attempted to unfilter instance %s which is not ' 'filtered'), instance['id']) -- cgit From 74bae1b1e2b298ef8425f7cb1aefd3826db40147 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Thu, 19 May 2011 13:50:11 -0700 Subject: Separate out tests for when unfilter is called from iptables vs. nwfilter driver. Re: lp783705 --- nova/tests/test_virt.py | 65 ++++++++++++++++++++++++++++++++++------------- nova/virt/libvirt_conn.py | 22 ++++++++-------- 2 files changed, 58 insertions(+), 29 deletions(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index babb5de9b..3b5a3867d 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -657,6 +657,21 @@ class LibvirtConnTestCase(test.TestCase): super(LibvirtConnTestCase, self).tearDown() +class FakeNWFilter: + def __init__(self): + self.undefine_call_count = 0 + + def undefine(self): + self.undefine_call_count += 1 + pass + + def _nwfilterLookupByName(self, ignore): + return self + + def _filterDefineXMLMock(self, xml): + return True + + class IptablesFirewallTestCase(test.TestCase): def setUp(self): super(IptablesFirewallTestCase, self).setUp() @@ -869,6 +884,35 @@ class IptablesFirewallTestCase(test.TestCase): self.assertEquals(ipv6_network_rules, ipv6_rules_per_network * networks_count) + def test_unfilter_instance_undefines_nwfilters(self): + admin_ctxt = context.get_admin_context() + + fakefilter = FakeNWFilter() + self.fw.nwfilter._conn.nwfilterDefineXML =\ + fakefilter._filterDefineXMLMock + self.fw.nwfilter._conn.nwfilterLookupByName =\ + fakefilter._nwfilterLookupByName + + instance_ref = self._create_instance_ref() + inst_id = instance_ref['id'] + instance = db.instance_get(self.context, inst_id) + + ip = '10.11.12.13' + network_ref = db.project_get_network(self.context, 'fake') + fixed_ip = {'address': ip, 'network_id': network_ref['id']} + db.fixed_ip_create(admin_ctxt, fixed_ip) + db.fixed_ip_update(admin_ctxt, ip, {'allocated': True, + 'instance_id': inst_id}) + self.fw.setup_basic_filtering(instance) + self.fw.prepare_instance_filter(instance) + self.fw.apply_instance_filter(instance) + self.fw.unfilter_instance(instance) + + # should attempt to undefine just the instance filter + self.assertEquals(fakefilter.undefine_call_count, 1) + + db.instance_destroy(admin_ctxt, instance_ref['id']) + class NWFilterTestCase(test.TestCase): def setUp(self): @@ -1047,26 +1091,11 @@ class NWFilterTestCase(test.TestCase): self.assertEquals(len(result), 3) def test_unfilter_instance_undefines_nwfilters(self): - class FakeNWFilter: - def __init__(self): - self.undefine_call_count = 0 - - def undefine(self): - self.undefine_call_count += 1 - pass - - fakefilter = FakeNWFilter() - - def _nwfilterLookupByName(ignore): - return fakefilter - - def _filterDefineXMLMock(xml): - return True - admin_ctxt = context.get_admin_context() - self.fw._conn.nwfilterDefineXML = _filterDefineXMLMock - self.fw._conn.nwfilterLookupByName = _nwfilterLookupByName + fakefilter = FakeNWFilter() + self.fw._conn.nwfilterDefineXML = fakefilter._filterDefineXMLMock + self.fw._conn.nwfilterLookupByName = fakefilter._nwfilterLookupByName instance_ref = self._create_instance() inst_id = instance_ref['id'] diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 706973176..f808a4b7b 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1834,7 +1834,7 @@ class NWFilterFirewall(FirewallDriver): # execute in a native thread and block current greenthread until done tpool.execute(self._conn.nwfilterDefineXML, xml) - def unfilter_instance(self, instance): + def unfilter_instance(self, instance, remove_secgroup=True): """Clear out the nwfilter rules.""" network_info = _get_network_info(instance) instance_name = instance.name @@ -1846,19 +1846,19 @@ class NWFilterFirewall(FirewallDriver): self._conn.nwfilterLookupByName(instance_filter_name).\ undefine() except libvirt.libvirtError: - LOG.debug(_('The nwfilter(%(instance_filter_name)s) for ' - '%(instance_name)s is not found.') % locals()) + LOG.debug(_('The nwfilter(%(instance_filter_name)s) ' + 'for %(instance_name)s is not found.') % locals()) instance_secgroup_filter_name =\ '%s-secgroup' % (self._instance_filter_name(instance)) - try: - self._conn.nwfilterLookupByName(instance_secgroup_filter_name).\ - undefine() - except libvirt.libvirtError: - # This will happen if called by IptablesFirewallDriver - LOG.debug(_('The nwfilter(%(instance_secgroup_filter_name)s) for ' - '%(instance_name)s is not found.') % locals()) + if remove_secgroup: + try: + self._conn.nwfilterLookupByName(instance_secgroup_filter_name)\ + .undefine() + except libvirt.libvirtError: + LOG.debug(_('The nwfilter(%(instance_secgroup_filter_name)s) ' + 'for %(instance_name)s is not found.') % locals()) def prepare_instance_filter(self, instance, network_info=None): """ @@ -2022,7 +2022,7 @@ class IptablesFirewallDriver(FirewallDriver): if self.instances.pop(instance['id'], None): self.remove_filters_for_instance(instance) self.iptables.apply() - self.nwfilter.unfilter_instance(instance) + self.nwfilter.unfilter_instance(instance, False) else: LOG.info(_('Attempted to unfilter instance %s which is not ' 'filtered'), instance['id']) -- cgit From 0bb2d0085e1fb3ba22a408f405f4539aa07b226c Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Fri, 20 May 2011 08:59:07 -0700 Subject: make nwfilter mock more 'realistic' by having it remember which filters have been defined --- nova/tests/test_virt.py | 56 +++++++++++++++++++++++++++++++++++++++-------- nova/virt/libvirt_conn.py | 17 +++++++------- 2 files changed, 55 insertions(+), 18 deletions(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 4bc5fed16..5e85e3a2f 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -659,16 +659,26 @@ class LibvirtConnTestCase(test.TestCase): class FakeNWFilter: def __init__(self): - self.undefine_call_count = 0 + self.filters = {} - def undefine(self): - self.undefine_call_count += 1 - pass - - def _nwfilterLookupByName(self, ignore): - return self + def _nwfilterLookupByName(self, name): + if name in self.filters: + return self.filters[name] + raise libvirt.libvirtError('Filter Not Found') def _filterDefineXMLMock(self, xml): + class FakeNWFilterInternal: + def __init__(self, parent, name): + self.name = name + self.parent = parent + + def undefine(self): + del self.parent.filters[self.name] + pass + tree = xml_to_tree(xml) + name = tree.get('name') + if name not in self.filters: + self.filters[name] = FakeNWFilterInternal(self, name) return True @@ -689,6 +699,20 @@ class IptablesFirewallTestCase(test.TestCase): self.fw = libvirt_conn.IptablesFirewallDriver( get_connection=lambda: self.fake_libvirt_connection) + def lazy_load_library_exists(self): + """check if libvirt is available.""" + # try to connect libvirt. if fail, skip test. + try: + import libvirt + import libxml2 + except ImportError: + return False + global libvirt + libvirt = __import__('libvirt') + libvirt_conn.libvirt = __import__('libvirt') + libvirt_conn.libxml2 = __import__('libxml2') + return True + def tearDown(self): self.manager.delete_project(self.project) self.manager.delete_user(self.user) @@ -895,6 +919,10 @@ class IptablesFirewallTestCase(test.TestCase): self.fw.do_refresh_security_group_rules("fake") def test_unfilter_instance_undefines_nwfilter(self): + # Skip if non-libvirt environment + if not self.lazy_load_library_exists(): + return + admin_ctxt = context.get_admin_context() fakefilter = FakeNWFilter() @@ -916,10 +944,11 @@ class IptablesFirewallTestCase(test.TestCase): self.fw.setup_basic_filtering(instance) self.fw.prepare_instance_filter(instance) self.fw.apply_instance_filter(instance) + original_filter_count = len(fakefilter.filters) self.fw.unfilter_instance(instance) # should attempt to undefine just the instance filter - self.assertEquals(fakefilter.undefine_call_count, 1) + self.assertEqual(original_filter_count - len(fakefilter.filters), 1) db.instance_destroy(admin_ctxt, instance_ref['id']) @@ -1109,6 +1138,12 @@ class NWFilterTestCase(test.TestCase): instance_ref = self._create_instance() inst_id = instance_ref['id'] + + self.security_group = self.setup_and_return_security_group() + + db.instance_add_security_group(self.context, inst_id, + self.security_group.id) + instance = db.instance_get(self.context, inst_id) ip = '10.11.12.13' @@ -1120,9 +1155,12 @@ class NWFilterTestCase(test.TestCase): self.fw.setup_basic_filtering(instance) self.fw.prepare_instance_filter(instance) self.fw.apply_instance_filter(instance) + original_filter_count = len(fakefilter.filters) + print fakefilter.filters.keys() self.fw.unfilter_instance(instance) + print fakefilter.filters.keys() # should attempt to undefine 2 filters: instance and instance-secgroup - self.assertEquals(fakefilter.undefine_call_count, 2) + self.assertEqual(original_filter_count - len(fakefilter.filters), 2) db.instance_destroy(admin_ctxt, instance_ref['id']) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 9241c1d9e..f27398aa3 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1838,7 +1838,7 @@ class NWFilterFirewall(FirewallDriver): # execute in a native thread and block current greenthread until done tpool.execute(self._conn.nwfilterDefineXML, xml) - def unfilter_instance(self, instance, remove_secgroup=True): + def unfilter_instance(self, instance): """Clear out the nwfilter rules.""" network_info = _get_network_info(instance) instance_name = instance.name @@ -1856,13 +1856,12 @@ class NWFilterFirewall(FirewallDriver): instance_secgroup_filter_name =\ '%s-secgroup' % (self._instance_filter_name(instance)) - if remove_secgroup: - try: - self._conn.nwfilterLookupByName(instance_secgroup_filter_name)\ - .undefine() - except libvirt.libvirtError: - LOG.debug(_('The nwfilter(%(instance_secgroup_filter_name)s) ' - 'for %(instance_name)s is not found.') % locals()) + try: + self._conn.nwfilterLookupByName(instance_secgroup_filter_name)\ + .undefine() + except libvirt.libvirtError: + LOG.debug(_('The nwfilter(%(instance_secgroup_filter_name)s) ' + 'for %(instance_name)s is not found.') % locals()) def prepare_instance_filter(self, instance, network_info=None): """ @@ -2028,7 +2027,7 @@ class IptablesFirewallDriver(FirewallDriver): if self.instances.pop(instance['id'], None): self.remove_filters_for_instance(instance) self.iptables.apply() - self.nwfilter.unfilter_instance(instance, False) + self.nwfilter.unfilter_instance(instance) else: LOG.info(_('Attempted to unfilter instance %s which is not ' 'filtered'), instance['id']) -- cgit From 5c205bb5ef1565db4e52af538cf0d6b73cbeda37 Mon Sep 17 00:00:00 2001 From: Anthony Young Date: Fri, 20 May 2011 09:09:03 -0700 Subject: fix comments --- nova/tests/test_virt.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 5e85e3a2f..90c6de5a9 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -659,6 +659,7 @@ class LibvirtConnTestCase(test.TestCase): class FakeNWFilter: def __init__(self): + self.filters = {} def _nwfilterLookupByName(self, name): @@ -947,7 +948,7 @@ class IptablesFirewallTestCase(test.TestCase): original_filter_count = len(fakefilter.filters) self.fw.unfilter_instance(instance) - # should attempt to undefine just the instance filter + # should undefine just the instance filter self.assertEqual(original_filter_count - len(fakefilter.filters), 1) db.instance_destroy(admin_ctxt, instance_ref['id']) @@ -1160,7 +1161,7 @@ class NWFilterTestCase(test.TestCase): self.fw.unfilter_instance(instance) print fakefilter.filters.keys() - # should attempt to undefine 2 filters: instance and instance-secgroup + # should undefine 2 filters: instance and instance-secgroup self.assertEqual(original_filter_count - len(fakefilter.filters), 2) db.instance_destroy(admin_ctxt, instance_ref['id']) -- cgit From 7a521f49f6daf0a0a37a9ef98ff1ea8813f04a6f Mon Sep 17 00:00:00 2001 From: John Tran Date: Mon, 23 May 2011 14:54:11 -0700 Subject: merged from trunk --- nova/tests/test_cloud.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 50 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index ebfb5ee44..f3887b07b 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -37,7 +37,6 @@ from nova import rpc from nova import service from nova import test from nova import utils -from nova import exception from nova.auth import manager from nova.compute import power_state from nova.api.ec2 import cloud @@ -279,6 +278,26 @@ class CloudTestCase(test.TestCase): user_group=['all']) self.assertEqual(True, result['is_public']) + def test_deregister_image(self): + deregister_image = self.cloud.deregister_image + + def fake_delete(self, context, id): + return None + + self.stubs.Set(local.LocalImageService, 'delete', fake_delete) + # valid image + result = deregister_image(self.context, 'ami-00000001') + self.assertEqual(result['imageId'], 'ami-00000001') + # invalid image + self.stubs.UnsetAll() + + def fake_detail_empty(self, context): + return [] + + self.stubs.Set(local.LocalImageService, 'detail', fake_detail_empty) + self.assertRaises(exception.ImageNotFound, deregister_image, + self.context, 'ami-bad001') + def test_console_output(self): instance_type = FLAGS.default_instance_type max_count = 1 @@ -334,6 +353,36 @@ class CloudTestCase(test.TestCase): self.assertTrue(filter(lambda k: k['keyName'] == 'test1', keys)) self.assertTrue(filter(lambda k: k['keyName'] == 'test2', keys)) + def test_import_public_key(self): + # test when user provides all values + result1 = self.cloud.import_public_key(self.context, + 'testimportkey1', + 'mytestpubkey', + 'mytestfprint') + self.assertTrue(result1) + keydata = db.key_pair_get(self.context, + self.context.user.id, + 'testimportkey1') + self.assertEqual('mytestpubkey', keydata['public_key']) + self.assertEqual('mytestfprint', keydata['fingerprint']) + # test when user omits fingerprint + pubkey_path = os.path.join(os.path.dirname(__file__), 'public_key') + f = open(pubkey_path + '/dummy.pub', 'r') + dummypub = f.readline().rstrip() + f.close + f = open(pubkey_path + '/dummy.fingerprint', 'r') + dummyfprint = f.readline().rstrip() + f.close + result2 = self.cloud.import_public_key(self.context, + 'testimportkey2', + dummypub) + self.assertTrue(result2) + keydata = db.key_pair_get(self.context, + self.context.user.id, + 'testimportkey2') + self.assertEqual(dummypub, keydata['public_key']) + self.assertEqual(dummyfprint, keydata['fingerprint']) + def test_delete_key_pair(self): self._create_key('test') self.cloud.delete_key_pair(self.context, 'test') -- cgit From 884b6d3ed74c5a5f766e405ac2178066314fb6d3 Mon Sep 17 00:00:00 2001 From: William Wolf Date: Tue, 24 May 2011 09:51:21 -0400 Subject: make _make_fixture respect name passed in --- nova/tests/api/openstack/test_images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 2c329f920..82bf66e49 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -127,7 +127,7 @@ class _BaseImageServiceTests(test.TestCase): @staticmethod def _make_fixture(name): - fixture = {'name': 'test image', + fixture = {'name': name, 'updated': None, 'created': None, 'status': None, -- cgit From 8e7c3121fab4b5a87c2efe865f3c06b1bd267cbc Mon Sep 17 00:00:00 2001 From: John Tran Date: Tue, 24 May 2011 08:59:02 -0700 Subject: added imageid string to exception, per peer review --- nova/tests/test_cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index f3887b07b..e37aca4d6 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -226,7 +226,7 @@ class CloudTestCase(test.TestCase): 'type': 'machine'}}] def fake_show_none(meh, context, id): - raise exception.ImageNotFound + raise exception.ImageNotFound('bad_image_id') self.stubs.Set(local.LocalImageService, 'detail', fake_detail) # list all -- cgit From a0cffc4de8ba4b15958e320308477d42287858e7 Mon Sep 17 00:00:00 2001 From: John Tran Date: Tue, 24 May 2011 09:43:52 -0700 Subject: specified image_id keyword in exception arg --- nova/api/ec2/cloud.py | 2 +- nova/tests/test_cloud.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 59e00781e..80c61d62b 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -848,7 +848,7 @@ class CloudController(object): kwargs['ramdisk_id'] = ramdisk['id'] image = self._get_image(context, kwargs['image_id']) if not image: - raise exception.ImageNotFound(kwargs['image_id']) + raise exception.ImageNotFound(image_id=kwargs['image_id']) try: available = (image['properties']['image_state'] == 'available') except KeyError: diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index e37aca4d6..1e583377b 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -226,7 +226,7 @@ class CloudTestCase(test.TestCase): 'type': 'machine'}}] def fake_show_none(meh, context, id): - raise exception.ImageNotFound('bad_image_id') + raise exception.ImageNotFound(image_id='bad_image_id') self.stubs.Set(local.LocalImageService, 'detail', fake_detail) # list all -- cgit From f3d7ec3fd2b2b987ae1118a6ae96874e8bbfdac5 Mon Sep 17 00:00:00 2001 From: William Wolf Date: Tue, 24 May 2011 15:16:07 -0400 Subject: initial use of limited_by_marker --- nova/api/openstack/images.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index 34d4c27fc..b06429943 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -153,3 +153,25 @@ class ControllerV11(Controller): def get_default_xmlns(self, req): return common.XML_NS_V11 + + def index(self, req): + """Return an index listing of images available to the request. + + :param req: `wsgi.Request` object + """ + context = req.environ['nova.context'] + images = self._image_service.index(context) + images = common.limited_by_marker(images, req) + builder = self.get_builder(req).build + return dict(images=[builder(image, detail=False) for image in images]) + + def detail(self, req): + """Return a detailed index listing of images available to the request. + + :param req: `wsgi.Request` object. + """ + context = req.environ['nova.context'] + images = self._image_service.detail(context) + images = common.limited_by_marker(images, req) + builder = self.get_builder(req).build + return dict(images=[builder(image, detail=True) for image in images]) -- cgit From 0b9ede226674b253f638b78cdce5fa40b2991701 Mon Sep 17 00:00:00 2001 From: William Wolf Date: Wed, 25 May 2011 11:21:46 -0400 Subject: simplified the limiting differences for different versions of the API --- nova/api/openstack/images.py | 30 +++++++----------------------- 1 file changed, 7 insertions(+), 23 deletions(-) diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index b06429943..c96b1c3e3 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -53,6 +53,9 @@ class Controller(common.OpenstackController): self._compute_service = compute_service or compute.API() self._image_service = image_service or _default_service + def _limit_items(self, items, req): + return common.limited(items, req) + def index(self, req): """Return an index listing of images available to the request. @@ -60,7 +63,7 @@ class Controller(common.OpenstackController): """ context = req.environ['nova.context'] images = self._image_service.index(context) - images = common.limited(images, req) + images = self._limit_items(images, req) builder = self.get_builder(req).build return dict(images=[builder(image, detail=False) for image in images]) @@ -71,7 +74,7 @@ class Controller(common.OpenstackController): """ context = req.environ['nova.context'] images = self._image_service.detail(context) - images = common.limited(images, req) + images = self._limited_items(images, req) builder = self.get_builder(req).build return dict(images=[builder(image, detail=True) for image in images]) @@ -154,24 +157,5 @@ class ControllerV11(Controller): def get_default_xmlns(self, req): return common.XML_NS_V11 - def index(self, req): - """Return an index listing of images available to the request. - - :param req: `wsgi.Request` object - """ - context = req.environ['nova.context'] - images = self._image_service.index(context) - images = common.limited_by_marker(images, req) - builder = self.get_builder(req).build - return dict(images=[builder(image, detail=False) for image in images]) - - def detail(self, req): - """Return a detailed index listing of images available to the request. - - :param req: `wsgi.Request` object. - """ - context = req.environ['nova.context'] - images = self._image_service.detail(context) - images = common.limited_by_marker(images, req) - builder = self.get_builder(req).build - return dict(images=[builder(image, detail=True) for image in images]) + def _limit_items(self, items, req): + return common.limited_by_marker(items, req) -- cgit From c9926b12f4c554d9a21c6e77fc657e54a2dd4888 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Sun, 29 May 2011 18:01:46 -0700 Subject: starting --- doc/source/devref/distributed_scheduler.rst | 164 ++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100644 doc/source/devref/distributed_scheduler.rst diff --git a/doc/source/devref/distributed_scheduler.rst b/doc/source/devref/distributed_scheduler.rst new file mode 100644 index 000000000..75a4d57ce --- /dev/null +++ b/doc/source/devref/distributed_scheduler.rst @@ -0,0 +1,164 @@ +.. + Copyright 2011 OpenStack LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + +Distributed Scheduler +===== + +The Scheduler is akin to a Dating Service. Requests for the creation of new instances come in and Compute nodes are selected for where the work should be performed. In a small deployment we may be happy with the currently available Change Scheduler which randomly selects a Host from the available pool. Or if you need something a little more fancy you may want to use the Availability Zone Scheduler, which selects Compute hosts from a logical partitioning of available hosts (within a single Zone). + +But for larger deployments a more complex scheduling algorithm is required. Additionally, if you are using Zones in your Nova setup, you'll need a scheduler that understand how to pass instance requests from Zone to Zone. + +This is the purpose of the Distributed Scheduler (DS). The DS utilizes the Capabilities of a Zone and its component services to make informed decisions on where a new instance should be created. When making this decision it consults not only all the Compute nodes in the current Zone, but the Compute nodes in each Child Zone. This continues recursively until the ideal host is found. + +So, how does this all work? + +This document will explain the strategy employed by the ZoneAwareScheduler and its derivations. + +Costs & Weights +---------- +When deciding where to place an Instance, we compare a Weighted Cost for each Host. The Weighting, currently, is just the sum of each Cost. Costs are nothing more than integers from `0 - max_int`. Costs are computed by looking at the various Capabilities of the Host relative to the specs of the Instance being asked for. Trying to put an Instance with 8G of RAM on a Host that only has 4G remaining would have a very high cost. But putting a 512m RAM instance on an empty Host should have a low cost. + +Some Costs are more esoteric. Consider a rule that says we should prefer Hosts that don't already have an instance on it that is owned by the user requesting it (to mitigate against machine failures). Here we have to look at all the other Instances on the host to compute our cost. + +An example of some other costs might include selecting: +* a GPU-based host over a standard CPU +* a host with fast ethernet over a 10mbps line +* a host than can run Windows instances +* a host in the EU vs North America +* etc + +This Weight is computed for each Instance requested. If the customer asked for 1000 instances, the consumed resources on each Host are "virtually" depleted so the Cost can change accordingly. + +nova.scheduler.zone_aware_scheduler.ZoneAwareScheduler +----------- +As we explained in the Zones documentation, each Scheduler has a `ZoneManager` object that collects "Capabilities" about Child Zones and each of the Services running in the current Zone. The `ZoneAwareScheduler` uses this information to make its decisions. + +Here is how it works: + +1. The Compute nodes are Filtered and the remaining set are Weighted. +1a. Filtering the hosts is a simple matter of ensuring the Compute node has ample resources (CPU, RAM, DISK, etc) to fulfil the request. +1b. Weighing of the remaining Compute nodes is performed +2. The same request is sent to each Child Zone and step #1 is done there too. The resulting Weighted List is returned to the parent. +3. The Parent Zone sorts and aggregates all the Weights and a final Build Plan is constructed. +4. The Build Plan is executed upon. Concurrently, Instance Create requests are sent to each of the selected Hosts, be they local or in a child zone. Child Zones may forward the requests to their Child Zones as needed. + +Filtering and Weighing +------------ +Filtering (excluding Compute nodes incapable of fulfilling the request) and Weighing (computing the relative "fitness" of a Compute node to fulfill the request) are very subjective operations. + + + + + + +- +Routing between Zones is based on the Capabilities of that Zone. Capabilities are nothing more than key/value pairs. Values are multi-value, with each value separated with a semicolon (`;`). When expressed as a string they take the form: + +:: + + key=value;value;value, key=value;value;value + +Zones have Capabilities which are general to the Zone and are set via `--zone_capabilities` flag. Zones also have dynamic per-service Capabilities. Services derived from `nova.manager.SchedulerDependentManager` (such as Compute, Volume and Network) can set these capabilities by calling the `update_service_capabilities()` method on their `Manager` base class. These capabilities will be periodically sent to the Scheduler service automatically. The rate at which these updates are sent is controlled by the `--periodic_interval` flag. + +Flow within a Zone +------------------ +The brunt of the work within a Zone is done in the Scheduler Service. The Scheduler is responsible for: +- collecting capability messages from the Compute, Volume and Network nodes, +- polling the child Zones for their status and +- providing data to the Distributed Scheduler for performing load balancing calculations + +Inter-service communication within a Zone is done with RabbitMQ. Each class of Service (Compute, Volume and Network) has both a named message exchange (particular to that host) and a general message exchange (particular to that class of service). Messages sent to these exchanges are picked off in round-robin fashion. Zones introduce a new fan-out exchange per service. Messages sent to the fan-out exchange are picked up by all services of a particular class. This fan-out exchange is used by the Scheduler services to receive capability messages from the Compute, Volume and Network nodes. + +These capability messages are received by the Scheduler services and stored in the `ZoneManager` object. The SchedulerManager object has a reference to the `ZoneManager` it can use for load balancing. + +The `ZoneManager` also polls the child Zones periodically to gather their capabilities to aid in decision making. This is done via the OpenStack API `/v1.0/zones/info` REST call. This also captures the name of each child Zone. The Zone name is set via the `--zone_name` flag (and defaults to "nova"). + +Zone administrative functions +----------------------------- +Zone administrative operations are usually done using python-novaclient_ + +.. _python-novaclient: https://github.com/rackspace/python-novaclient + +In order to use the Zone operations, be sure to enable administrator operations in OpenStack API by setting the `--allow_admin_api=true` flag. + +Finally you need to enable Zone Forwarding. This will be used by the Distributed Scheduler initiative currently underway. Set `--enable_zone_routing=true` to enable this feature. + +Find out about this Zone +------------------------ +In any Zone you can find the Zone's name and capabilities with the ``nova zone-info`` command. + +:: + + alice@novadev:~$ nova zone-info + +-----------------+---------------+ + | Property | Value | + +-----------------+---------------+ + | compute_cpu | 0.7,0.7 | + | compute_disk | 123000,123000 | + | compute_network | 800,800 | + | hypervisor | xenserver | + | name | nova | + | network_cpu | 0.7,0.7 | + | network_disk | 123000,123000 | + | network_network | 800,800 | + | os | linux | + +-----------------+---------------+ + +This equates to a GET operation on `.../zones/info`. If you have no child Zones defined you'll usually only get back the default `name`, `hypervisor` and `os` capabilities. Otherwise you'll get back a tuple of min, max values for each capabilities of all the hosts of all the services running in the child zone. These take the `_ = ,` format. + +Adding a child Zone +------------------- +Any Zone can be a parent Zone. Children are associated to a Zone. The Zone where this command originates from is known as the Parent Zone. Routing is only ever conducted from a Zone to its children, never the other direction. From a parent zone you can add a child zone with the following command: + +:: + + nova zone-add + +You can get the `child zone api url`, `nova api key` and `username` from the `novarc` file in the child zone. For example: + +:: + + export NOVA_API_KEY="3bd1af06-6435-4e23-a827-413b2eb86934" + export NOVA_USERNAME="alice" + export NOVA_URL="http://192.168.2.120:8774/v1.0/" + + +This equates to a POST operation to `.../zones/` to add a new zone. No connection attempt to the child zone is done when this command. It only puts an entry in the db at this point. After about 30 seconds the `ZoneManager` in the Scheduler services will attempt to talk to the child zone and get its information. + +Getting a list of child Zones +----------------------------- + +:: + + nova zone-list + + alice@novadev:~$ nova zone-list + +----+-------+-----------+--------------------------------------------+---------------------------------+ + | ID | Name | Is Active | Capabilities | API URL | + +----+-------+-----------+--------------------------------------------+---------------------------------+ + | 2 | zone1 | True | hypervisor=xenserver;kvm, os=linux;windows | http://192.168.2.108:8774/v1.0/ | + | 3 | zone2 | True | hypervisor=xenserver;kvm, os=linux;windows | http://192.168.2.115:8774/v1.0/ | + +----+-------+-----------+--------------------------------------------+---------------------------------+ + +This equates to a GET operation to `.../zones`. + +Removing a child Zone +--------------------- +:: + + nova zone-delete + +This equates to a DELETE call to `.../zones/N`. The Zone with ID=N will be removed. This will only remove the zone entry from the current (parent) Zone, no child Zones are affected. Removing a Child Zone doesn't affect any other part of the hierarchy. -- cgit From 5aa54545486ffe9d9988761576f497de9a957d47 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Sun, 29 May 2011 23:42:46 -0300 Subject: lots more --- doc/source/devref/distributed_scheduler.rst | 44 +++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 6 deletions(-) diff --git a/doc/source/devref/distributed_scheduler.rst b/doc/source/devref/distributed_scheduler.rst index 75a4d57ce..7599f2cc5 100644 --- a/doc/source/devref/distributed_scheduler.rst +++ b/doc/source/devref/distributed_scheduler.rst @@ -25,7 +25,7 @@ This is the purpose of the Distributed Scheduler (DS). The DS utilizes the Capab So, how does this all work? -This document will explain the strategy employed by the ZoneAwareScheduler and its derivations. +This document will explain the strategy employed by the `ZoneAwareScheduler` and its derivations. You should read the Zones documentation before reading this. Costs & Weights ---------- @@ -48,19 +48,51 @@ As we explained in the Zones documentation, each Scheduler has a `ZoneManager` o Here is how it works: -1. The Compute nodes are Filtered and the remaining set are Weighted. +1. The Compute nodes are Filtered and the nodes remaining are Weighed. 1a. Filtering the hosts is a simple matter of ensuring the Compute node has ample resources (CPU, RAM, DISK, etc) to fulfil the request. -1b. Weighing of the remaining Compute nodes is performed +1b. Weighing of the remaining Compute nodes assigns a number based on their suitability for the request. 2. The same request is sent to each Child Zone and step #1 is done there too. The resulting Weighted List is returned to the parent. 3. The Parent Zone sorts and aggregates all the Weights and a final Build Plan is constructed. 4. The Build Plan is executed upon. Concurrently, Instance Create requests are sent to each of the selected Hosts, be they local or in a child zone. Child Zones may forward the requests to their Child Zones as needed. Filtering and Weighing ------------ -Filtering (excluding Compute nodes incapable of fulfilling the request) and Weighing (computing the relative "fitness" of a Compute node to fulfill the request) are very subjective operations. - - +Filtering (excluding Compute nodes incapable of fulfilling the request) and Weighing (computing the relative "fitness" of a Compute node to fulfill the request) are very subjective operations. Service Providers will probably have a very different set of filtering and weighing rules than private cloud administrators. The filtering and weighing aspects of the `ZoneAwareScheduler` are flexible and extensible. We will explain how to do this later in this document. +Requesting a new instance +------------ +To request a new instance, a call is made to `nova.compute.api.create()`. The type of instance created depends on the value of the `InstanceType` record being passed in. The `InstanceType` determines the amount of disk, cpu, ram and network required for the instance. Administrators can add new `InstanceType` records to suit their needs. For more complicated instance requests we need to go beyond the default fields in the `InstanceType` table, but we'll discuss that later. + +`nova.compute.api.create()` performs the following actions: +1. it validates all the fields passed into it. +2. it creates an entry in the `Instance` table for each instance requested +3. it puts one `run_instance` message in the scheduler queue for each instance requested +4. the schedulers pick off the messages and decide which Compute node should handle the request. +5. the `run_instance` message is forwarded to the Compute node for processing and the instance is created. +6. it returns a list of dicts representing each of the `Instance` records (even if the instance has not been activated yet). At least the `instance_id`'s are valid. + +Generally, the standard schedulers (like `ChangeScheduler` and `AvailabilityZoneScheduler`) only operate in the current Zone. They have no concept of Child Zones. + +The problem with this approach is that each request is scattered amongst each of the schedulers. If we are asking for 1000 instances, each scheduler gets the requests one-at-a-time. There is no possability of optimizing the requests to take into account all 1000 instances as a group. We call this Single-Shot vs. All-at-Once. + +For the `ZoneAwareScheduler` we need to use the All-at-Once approach. We need to consider all the hosts across all the Zones before deciding where they should reside. In order to handle this we have a new method `nova.compute.api.create_all_at_once()`. This method does things a little differently: +1. it validates all the fields passed into it. +2. it creates a single `request_id` for all of instances created. This is a UUID. +3. it creates a single `run_instance` request in the scheduler queue +4. a scheduler picks the message off the queue and works on it. +5. the scheduler sends off an OS API `POST /zones/select` command to each Child Zone. The `BODY` payload of the call contains the `request_spec`. +6. the Child Zones use the `request_spec` to compute a weighted list for each instance requested. No attempt to actually create an instance is done at this point. We're only estimating the suitability of the Zones. +7. if the Child Zone has its own Child Zone's, the `/zones/select` call will be sent down to them as well. +8. Finally, when all the estimates have bubbled back to the Zone that initiated the call, all the results are merged, sorted and processed. +9. Now the instances can be created. The initiating Zone either forwards the `run_instance` message to the local Compute node to do the work, or it issues a `POST /servers` call to the relevant Child Zone. The parameters to the Child Zone call are the same as what was passed in by the user. + +The Catch +------------- +This all seems pretty straightforward but, like most things, there's a catch. Zones are expected to operate in complete isolation from each other. Each Zone has its own AMQP service, Database and set of Nova Services. But, for security reasons Zones should never leak information about the architectural layout internally. That means Zones cannot leak information about hostnames or service IP addresses outside of its world. + +When `POST /zones/select` is called to estimate which Compute node to use, time passes until the `POST /servers` call is issued. If we only passed the Weight back from the `select` we would have to re-compute the appropriate Compute node for the create command ... and we could end up with a different host. Somehow we need to remember the results of our computations and pass them outside of the Zone. Now, we could store this information in the local database and return a reference to it, but remember that the vast majority of weights are going be ignored. Storing them in the database would result in a flood of disk access and then we have to clean up all these entries periodically. Recall that there are going to be many many `select` calls issued to Child Zones asking for estimates. + +Instead, we take a rather innovative approach to the problem. We encrypt all the child zone internal details and pass them back the to parent Zone. If the parent zone decides to use a child Zone for the instance it simply passes the encrypted data back to the child during the `POST /servers` call as an extra parameter. The child Zone can then decrypt the hint and go directly to the Compute node previously selected. -- cgit From c3c2c1a63c126f046457d0d61306ebe9c46af700 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 30 May 2011 00:00:28 -0300 Subject: basic flow done --- doc/source/devref/distributed_scheduler.rst | 105 ++++------------------------ 1 file changed, 13 insertions(+), 92 deletions(-) diff --git a/doc/source/devref/distributed_scheduler.rst b/doc/source/devref/distributed_scheduler.rst index 7599f2cc5..c9aaf8c01 100644 --- a/doc/source/devref/distributed_scheduler.rst +++ b/doc/source/devref/distributed_scheduler.rst @@ -55,6 +55,8 @@ Here is how it works: 3. The Parent Zone sorts and aggregates all the Weights and a final Build Plan is constructed. 4. The Build Plan is executed upon. Concurrently, Instance Create requests are sent to each of the selected Hosts, be they local or in a child zone. Child Zones may forward the requests to their Child Zones as needed. +`ZoneAwareScheduler` by itself is not capable of handling all the provisioning itself. Derived classes are used to select which Host filtering and Weighing strategy will be used. We'll go into more detail on that later. + Filtering and Weighing ------------ Filtering (excluding Compute nodes incapable of fulfilling the request) and Weighing (computing the relative "fitness" of a Compute node to fulfill the request) are very subjective operations. Service Providers will probably have a very different set of filtering and weighing rules than private cloud administrators. The filtering and weighing aspects of the `ZoneAwareScheduler` are flexible and extensible. We will explain how to do this later in this document. @@ -77,7 +79,7 @@ The problem with this approach is that each request is scattered amongst each of For the `ZoneAwareScheduler` we need to use the All-at-Once approach. We need to consider all the hosts across all the Zones before deciding where they should reside. In order to handle this we have a new method `nova.compute.api.create_all_at_once()`. This method does things a little differently: 1. it validates all the fields passed into it. -2. it creates a single `request_id` for all of instances created. This is a UUID. +2. it creates a single `reservation_id` for all of instances created. This is a UUID. 3. it creates a single `run_instance` request in the scheduler queue 4. a scheduler picks the message off the queue and works on it. 5. the scheduler sends off an OS API `POST /zones/select` command to each Child Zone. The `BODY` payload of the call contains the `request_spec`. @@ -85,6 +87,7 @@ For the `ZoneAwareScheduler` we need to use the All-at-Once approach. We need to 7. if the Child Zone has its own Child Zone's, the `/zones/select` call will be sent down to them as well. 8. Finally, when all the estimates have bubbled back to the Zone that initiated the call, all the results are merged, sorted and processed. 9. Now the instances can be created. The initiating Zone either forwards the `run_instance` message to the local Compute node to do the work, or it issues a `POST /servers` call to the relevant Child Zone. The parameters to the Child Zone call are the same as what was passed in by the user. +10. The `reservation_id` is passed back to the caller. Later we explain how the user can check on the status of the command with this `reservation_id`. The Catch ------------- @@ -92,105 +95,23 @@ This all seems pretty straightforward but, like most things, there's a catch. Zo When `POST /zones/select` is called to estimate which Compute node to use, time passes until the `POST /servers` call is issued. If we only passed the Weight back from the `select` we would have to re-compute the appropriate Compute node for the create command ... and we could end up with a different host. Somehow we need to remember the results of our computations and pass them outside of the Zone. Now, we could store this information in the local database and return a reference to it, but remember that the vast majority of weights are going be ignored. Storing them in the database would result in a flood of disk access and then we have to clean up all these entries periodically. Recall that there are going to be many many `select` calls issued to Child Zones asking for estimates. -Instead, we take a rather innovative approach to the problem. We encrypt all the child zone internal details and pass them back the to parent Zone. If the parent zone decides to use a child Zone for the instance it simply passes the encrypted data back to the child during the `POST /servers` call as an extra parameter. The child Zone can then decrypt the hint and go directly to the Compute node previously selected. - - - -- -Routing between Zones is based on the Capabilities of that Zone. Capabilities are nothing more than key/value pairs. Values are multi-value, with each value separated with a semicolon (`;`). When expressed as a string they take the form: - -:: - - key=value;value;value, key=value;value;value - -Zones have Capabilities which are general to the Zone and are set via `--zone_capabilities` flag. Zones also have dynamic per-service Capabilities. Services derived from `nova.manager.SchedulerDependentManager` (such as Compute, Volume and Network) can set these capabilities by calling the `update_service_capabilities()` method on their `Manager` base class. These capabilities will be periodically sent to the Scheduler service automatically. The rate at which these updates are sent is controlled by the `--periodic_interval` flag. - -Flow within a Zone ------------------- -The brunt of the work within a Zone is done in the Scheduler Service. The Scheduler is responsible for: -- collecting capability messages from the Compute, Volume and Network nodes, -- polling the child Zones for their status and -- providing data to the Distributed Scheduler for performing load balancing calculations - -Inter-service communication within a Zone is done with RabbitMQ. Each class of Service (Compute, Volume and Network) has both a named message exchange (particular to that host) and a general message exchange (particular to that class of service). Messages sent to these exchanges are picked off in round-robin fashion. Zones introduce a new fan-out exchange per service. Messages sent to the fan-out exchange are picked up by all services of a particular class. This fan-out exchange is used by the Scheduler services to receive capability messages from the Compute, Volume and Network nodes. - -These capability messages are received by the Scheduler services and stored in the `ZoneManager` object. The SchedulerManager object has a reference to the `ZoneManager` it can use for load balancing. - -The `ZoneManager` also polls the child Zones periodically to gather their capabilities to aid in decision making. This is done via the OpenStack API `/v1.0/zones/info` REST call. This also captures the name of each child Zone. The Zone name is set via the `--zone_name` flag (and defaults to "nova"). - -Zone administrative functions ------------------------------ -Zone administrative operations are usually done using python-novaclient_ - -.. _python-novaclient: https://github.com/rackspace/python-novaclient - -In order to use the Zone operations, be sure to enable administrator operations in OpenStack API by setting the `--allow_admin_api=true` flag. - -Finally you need to enable Zone Forwarding. This will be used by the Distributed Scheduler initiative currently underway. Set `--enable_zone_routing=true` to enable this feature. - -Find out about this Zone ------------------------- -In any Zone you can find the Zone's name and capabilities with the ``nova zone-info`` command. - -:: - - alice@novadev:~$ nova zone-info - +-----------------+---------------+ - | Property | Value | - +-----------------+---------------+ - | compute_cpu | 0.7,0.7 | - | compute_disk | 123000,123000 | - | compute_network | 800,800 | - | hypervisor | xenserver | - | name | nova | - | network_cpu | 0.7,0.7 | - | network_disk | 123000,123000 | - | network_network | 800,800 | - | os | linux | - +-----------------+---------------+ - -This equates to a GET operation on `.../zones/info`. If you have no child Zones defined you'll usually only get back the default `name`, `hypervisor` and `os` capabilities. Otherwise you'll get back a tuple of min, max values for each capabilities of all the hosts of all the services running in the child zone. These take the `_ = ,` format. - -Adding a child Zone -------------------- -Any Zone can be a parent Zone. Children are associated to a Zone. The Zone where this command originates from is known as the Parent Zone. Routing is only ever conducted from a Zone to its children, never the other direction. From a parent zone you can add a child zone with the following command: - -:: - - nova zone-add - -You can get the `child zone api url`, `nova api key` and `username` from the `novarc` file in the child zone. For example: - -:: +Instead, we take a rather innovative approach to the problem. We encrypt all the child zone internal details and pass them back the to parent Zone. If the parent zone decides to use a child Zone for the instance it simply passes the encrypted data back to the child during the `POST /servers` call as an extra parameter. The child Zone can then decrypt the hint and go directly to the Compute node previously selected. If the estimate isn't used, it is simply discarded by the parent. - export NOVA_API_KEY="3bd1af06-6435-4e23-a827-413b2eb86934" - export NOVA_USERNAME="alice" - export NOVA_URL="http://192.168.2.120:8774/v1.0/" +In the case of nested child Zones, each Zone re-encrypts the weighted list results and passes those values to the parent. +Throughout the `nova.api.openstack.servers`, `nova.api.openstack.zones`, `nova.compute.api.create*` and `nova.scheduler.zone_aware_scheduler` code you'll see references to `blob` and `child_blob`. These are the encrypted hints about which Compute node to use. -This equates to a POST operation to `.../zones/` to add a new zone. No connection attempt to the child zone is done when this command. It only puts an entry in the db at this point. After about 30 seconds the `ZoneManager` in the Scheduler services will attempt to talk to the child zone and get its information. +Reservation ID's +--------------- -Getting a list of child Zones ------------------------------ -:: - nova zone-list - alice@novadev:~$ nova zone-list - +----+-------+-----------+--------------------------------------------+---------------------------------+ - | ID | Name | Is Active | Capabilities | API URL | - +----+-------+-----------+--------------------------------------------+---------------------------------+ - | 2 | zone1 | True | hypervisor=xenserver;kvm, os=linux;windows | http://192.168.2.108:8774/v1.0/ | - | 3 | zone2 | True | hypervisor=xenserver;kvm, os=linux;windows | http://192.168.2.115:8774/v1.0/ | - +----+-------+-----------+--------------------------------------------+---------------------------------+ +Host Filter +-------------- -This equates to a GET operation to `.../zones`. -Removing a child Zone ---------------------- -:: +Cost Scheduler Weighing +-------------- - nova zone-delete -This equates to a DELETE call to `.../zones/N`. The Zone with ID=N will be removed. This will only remove the zone entry from the current (parent) Zone, no child Zones are affected. Removing a Child Zone doesn't affect any other part of the hierarchy. -- cgit From 5101aa300b087bf57f22cb128649679e8b11051d Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 30 May 2011 00:45:15 -0300 Subject: reservation_id's done --- doc/source/devref/distributed_scheduler.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/doc/source/devref/distributed_scheduler.rst b/doc/source/devref/distributed_scheduler.rst index c9aaf8c01..402f50bee 100644 --- a/doc/source/devref/distributed_scheduler.rst +++ b/doc/source/devref/distributed_scheduler.rst @@ -104,8 +104,17 @@ Throughout the `nova.api.openstack.servers`, `nova.api.openstack.zones`, `nova.c Reservation ID's --------------- +NOTE: The features described in this section are related to the up-coming 'merge-4' branch. +The OpenStack API allows a user to list all the instances they own via the `GET /servers/` command or the details on a particular instance via `GET /servers/###`. This mechanism is usually sufficient since OS API only allows for creating one instance at a time, unlike the EC2 API which allows you to specify a quantity of instances to be created. +NOTE: currently the `GET /servers` command is not Zone-aware since all operations done in child Zones are done via a single administrative account. Therefore, asking a child Zone to `GET /servers` would return all the active instances ... and that would be bad. Later, when the Keystone Auth system is integrated with Nova, this functionality will be enabled. + +We could use the OS API 1.1 Extensions mechanism to accept a `num_instances` parameter, but this would result in a different return code. Instead of getting back an `Instance` record, we would be getting back a `reservation_id`. So, instead, we've implemented a new command `POST /zones/servers` command which is nearly identical to `POST /servers` except that it takes a `num_instances` parameter and returns a `reservation_id`. Perhaps in OS API 2.x we can unify these approaches. + +Finally, we need to give the user a way to get information on each of the instances created under this `reservation_id`. Fortunately, this is still possible with the existing `GET /servers` command, so long as we add a new optional `reservation_id` parameter. + +`python-novaclient` will be extended to support both of these changes. Host Filter -------------- -- cgit From 0cf5316131aecbac5e843282e2e2eb2acd3fc9e3 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Mon, 30 May 2011 05:03:45 -0700 Subject: first cut complete --- doc/source/devref/distributed_scheduler.rst | 41 +++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/doc/source/devref/distributed_scheduler.rst b/doc/source/devref/distributed_scheduler.rst index 402f50bee..a45505640 100644 --- a/doc/source/devref/distributed_scheduler.rst +++ b/doc/source/devref/distributed_scheduler.rst @@ -119,8 +119,49 @@ Finally, we need to give the user a way to get information on each of the instan Host Filter -------------- +As we mentioned earlier, filtering hosts is a very deployment specific process. Service Providers may have a different set of criteria for filtering Compute nodes than a University. To faciliate this the `nova.scheduler.host_filter` module supports a variety of filtering strategies as well as an easy means for plugging in your own algorithms. +The filter used is determined by the `--default_host_filter` flag, which points to a Python Class. By default this flag is set to `nova.scheduler.host_filter.AllHostsFilter` which simply returns all available hosts. But there are others. + +`nova.scheduler.host_filter.InstanceTypeFilter` provides host filtering based on the memory and disk size specified in the `InstanceType` record passed into `run_instance`. +`nova.scheduler.host_filter.JSONFilter` filters hosts based on simple JSON expression grammar. Using a LISP-like JSON structure the caller can request instances based on criteria well beyond what `InstanceType` specifies. See `nova.tests.test_host_filter` for examples. + +To create your own `HostFilter` the user simply has to derive from `nova.scheduler.host_filter.HostFilter` and implement two methods: `instance_type_to_filter` and `filter_hosts`. Since Nova is currently dependent on the `InstanceType` structure, the `instance_type_to_filter` method should take an `InstanceType` and turn it into an internal data structure usable by your filter. This is for backward compatibility with existing OpenStack and EC2 API calls. If you decide the create your own call for creating instances not based on `Flavors` or `InstanceTypes` you can ignore this method. The real work is done in `filter_hosts` which must return a list of weight tuples for each appropriate host. The set of all available hosts is in the `ZoneManager` object passed into the call as well as the filter query. The weight tuple contains (``, ``) where `` is whatever you want it to be. + Cost Scheduler Weighing -------------- +Every `ZoneAwareScheduler` derivation must also override the `weigh_hosts` method. This takes the list of filtered hosts (generated by the `filter_hosts` method) and returns a list of weight dicts. The weight dicts must contain two keys: `weight` and `hostname` where `weight` is simply an integer (lower is better) and `hostname` is the name of the host. The list does not need to be sorted, this will be done by the `ZoneAwareScheduler` base class when all the results have been assembled. + +Simple Zone Aware Scheduling +-------------- +The easiest way to get started with the Zone Aware Scheduler is to use the `nova.scheduler.host_filter.HostFilterScheduler`. This scheduler uses the default Host Filter as and the `weight_hosts` method simply returns a weight of 1 for all hosts. But, from this, you can see calls being routed from Zone to Zone and follow the flow of things. + +The `--scheduler_driver` flag is how you specify the Scheduler class name. + +Flags +-------------- + +All this Zone and Distributed Scheduler stuff can seem a little daunting to configure, but it's actually not too bad. Here are some of the main flags you should set in your `nova.conf` file: + +:: + --allow_admin_api=true + --enable_zone_routing=true + --zone_name=zone1 + --build_plan_encryption_key=c286696d887c9aa0611bbb3e2025a45b + --scheduler_driver=nova.scheduler.host_filter.HostFilterScheduler + --default_host_filter=nova.scheduler.host_filter.AllHostsFilter + +`--allow_admin_api` must be set for OS API to enable the new `/zones/*` commands. +`--enable_zone_routing` must be set for OS API commands such as `create()`, `pause()` and `delete()` to get routed from Zone to Zone when looking for instances. +`--zone_name` is only required in Child Zones. The default Zone name is `nova`, but you may want to name your child Zones something useful. Duplicate Zone names are not an issue. +`build_plan_encryption_key` is the SHA-256 key for encrypting/decrypting the Host information when it leaves a Zone. Be sure to change this key for each Zone you create. Do not duplicate keys. +`scheduler_driver` is the real work horse of the operation. For Distributed Scheduler, you need to specify a class derived from `nova.scheduler.zone_aware_scheduler.ZoneAwareScheduler` +`default_host_filter` is the host filter to be used for filtering candidate Compute nodes. + +Some optional flags which are handy for debugging are: +:: + --connection_type=fake + --verbose +Using the `Fake` virtualization driver is handy when you're setting this stuff up so you're not dealing with a million possible issues at once. When things seem to working correctly, switch back to whatever hypervisor your deployment uses. -- cgit From 1adb96550640a65a723635f2dc98e4595f95fd52 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Tue, 31 May 2011 08:26:11 -0700 Subject: edits based on ed's feedback --- doc/source/devref/distributed_scheduler.rst | 85 +++++++++++++++-------------- 1 file changed, 45 insertions(+), 40 deletions(-) diff --git a/doc/source/devref/distributed_scheduler.rst b/doc/source/devref/distributed_scheduler.rst index a45505640..28ba20af7 100644 --- a/doc/source/devref/distributed_scheduler.rst +++ b/doc/source/devref/distributed_scheduler.rst @@ -1,3 +1,7 @@ + + + + .. Copyright 2011 OpenStack LLC All Rights Reserved. @@ -17,7 +21,7 @@ Distributed Scheduler ===== -The Scheduler is akin to a Dating Service. Requests for the creation of new instances come in and Compute nodes are selected for where the work should be performed. In a small deployment we may be happy with the currently available Change Scheduler which randomly selects a Host from the available pool. Or if you need something a little more fancy you may want to use the Availability Zone Scheduler, which selects Compute hosts from a logical partitioning of available hosts (within a single Zone). +The Scheduler is akin to a Dating Service. Requests for the creation of new instances come in and the most applicable Compute nodes are selected from a large pool of potential candidates. In a small deployment we may be happy with the currently available Change Scheduler which randomly selects a Host from the available pool. Or if you need something a little more fancy you may want to use the Availability Zone Scheduler, which selects Compute hosts from a logical partitioning of available hosts (within a single Zone). But for larger deployments a more complex scheduling algorithm is required. Additionally, if you are using Zones in your Nova setup, you'll need a scheduler that understand how to pass instance requests from Zone to Zone. @@ -29,7 +33,7 @@ This document will explain the strategy employed by the `ZoneAwareScheduler` and Costs & Weights ---------- -When deciding where to place an Instance, we compare a Weighted Cost for each Host. The Weighting, currently, is just the sum of each Cost. Costs are nothing more than integers from `0 - max_int`. Costs are computed by looking at the various Capabilities of the Host relative to the specs of the Instance being asked for. Trying to put an Instance with 8G of RAM on a Host that only has 4G remaining would have a very high cost. But putting a 512m RAM instance on an empty Host should have a low cost. +When deciding where to place an Instance, we compare a Weighted Cost for each Host. The Weighting, currently, is just the sum of each Cost. Costs are nothing more than integers from `0 - max_int`. Costs are computed by looking at the various Capabilities of the Host relative to the specs of the Instance being asked for. Trying to putting a plain vanilla instance on a high performance host should have a very high cost. But putting a vanilla instance on a vanilla Host should have a low cost. Some Costs are more esoteric. Consider a rule that says we should prefer Hosts that don't already have an instance on it that is owned by the user requesting it (to mitigate against machine failures). Here we have to look at all the other Instances on the host to compute our cost. @@ -44,73 +48,73 @@ This Weight is computed for each Instance requested. If the customer asked for 1 nova.scheduler.zone_aware_scheduler.ZoneAwareScheduler ----------- -As we explained in the Zones documentation, each Scheduler has a `ZoneManager` object that collects "Capabilities" about Child Zones and each of the Services running in the current Zone. The `ZoneAwareScheduler` uses this information to make its decisions. +As we explained in the Zones documentation, each Scheduler has a `ZoneManager` object that collects "Capabilities" about child Zones and each of the services running in the current Zone. The `ZoneAwareScheduler` uses this information to make its decisions. Here is how it works: -1. The Compute nodes are Filtered and the nodes remaining are Weighed. -1a. Filtering the hosts is a simple matter of ensuring the Compute node has ample resources (CPU, RAM, DISK, etc) to fulfil the request. -1b. Weighing of the remaining Compute nodes assigns a number based on their suitability for the request. -2. The same request is sent to each Child Zone and step #1 is done there too. The resulting Weighted List is returned to the parent. -3. The Parent Zone sorts and aggregates all the Weights and a final Build Plan is constructed. -4. The Build Plan is executed upon. Concurrently, Instance Create requests are sent to each of the selected Hosts, be they local or in a child zone. Child Zones may forward the requests to their Child Zones as needed. +1. The compute nodes are filtered and the nodes remaining are weighed. +1a. Filtering the hosts is a simple matter of ensuring the compute node has ample resources (CPU, RAM, Disk, etc) to fulfil the request. +1b. Weighing of the remaining compute nodes assigns a number based on their suitability for the request. +2. The same request is sent to each child Zone and step #1 is done there too. The resulting weighted list is returned to the parent. +3. The parent Zone sorts and aggregates all the weights and a final build plan is constructed. +4. The build plan is executed upon. Concurrently, instance create requests are sent to each of the selected hosts, be they local or in a child zone. Child Zones may forward the requests to their child Zones as needed. -`ZoneAwareScheduler` by itself is not capable of handling all the provisioning itself. Derived classes are used to select which Host filtering and Weighing strategy will be used. We'll go into more detail on that later. +`ZoneAwareScheduler` by itself is not capable of handling all the provisioning itself. Derived classes are used to select which host filtering and weighing strategy will be used. Filtering and Weighing ------------ -Filtering (excluding Compute nodes incapable of fulfilling the request) and Weighing (computing the relative "fitness" of a Compute node to fulfill the request) are very subjective operations. Service Providers will probably have a very different set of filtering and weighing rules than private cloud administrators. The filtering and weighing aspects of the `ZoneAwareScheduler` are flexible and extensible. We will explain how to do this later in this document. +The filtering (excluding compute nodes incapable of fulfilling the request) and weighing (computing the relative "fitness" of a compute node to fulfill the request) rules used are very subjective operations ... Service Providers will probably have a very different set of filtering and weighing rules than private cloud administrators. The filtering and weighing aspects of the `ZoneAwareScheduler` are flexible and extensible. Requesting a new instance ------------ -To request a new instance, a call is made to `nova.compute.api.create()`. The type of instance created depends on the value of the `InstanceType` record being passed in. The `InstanceType` determines the amount of disk, cpu, ram and network required for the instance. Administrators can add new `InstanceType` records to suit their needs. For more complicated instance requests we need to go beyond the default fields in the `InstanceType` table, but we'll discuss that later. +Prior to the `ZoneAwareScheduler`, to request a new instance, a call was made to `nova.compute.api.create()`. The type of instance created depended on the value of the `InstanceType` record being passed in. The `InstanceType` determined the amount of disk, CPU, RAM and network required for the instance. Administrators can add new `InstanceType` records to suit their needs. For more complicated instance requests we need to go beyond the default fields in the `InstanceType` table. -`nova.compute.api.create()` performs the following actions: -1. it validates all the fields passed into it. -2. it creates an entry in the `Instance` table for each instance requested -3. it puts one `run_instance` message in the scheduler queue for each instance requested -4. the schedulers pick off the messages and decide which Compute node should handle the request. -5. the `run_instance` message is forwarded to the Compute node for processing and the instance is created. -6. it returns a list of dicts representing each of the `Instance` records (even if the instance has not been activated yet). At least the `instance_id`'s are valid. +`nova.compute.api.create()` performed the following actions: +1. it validated all the fields passed into it. +2. it created an entry in the `Instance` table for each instance requested +3. it put one `run_instance` message in the scheduler queue for each instance requested +4. the schedulers picked off the messages and decided which compute node should handle the request. +5. the `run_instance` message was forwarded to the compute node for processing and the instance is created. +6. it returned a list of dicts representing each of the `Instance` records (even if the instance has not been activated yet). At least the `instance_id`s are valid. -Generally, the standard schedulers (like `ChangeScheduler` and `AvailabilityZoneScheduler`) only operate in the current Zone. They have no concept of Child Zones. +Generally, the standard schedulers (like `ChanceScheduler` and `AvailabilityZoneScheduler`) only operate in the current Zone. They have no concept of child Zones. -The problem with this approach is that each request is scattered amongst each of the schedulers. If we are asking for 1000 instances, each scheduler gets the requests one-at-a-time. There is no possability of optimizing the requests to take into account all 1000 instances as a group. We call this Single-Shot vs. All-at-Once. +The problem with this approach is each request is scattered amongst each of the schedulers. If we are asking for 1000 instances, each scheduler gets the requests one-at-a-time. There is no possability of optimizing the requests to take into account all 1000 instances as a group. We call this Single-Shot vs. All-at-Once. For the `ZoneAwareScheduler` we need to use the All-at-Once approach. We need to consider all the hosts across all the Zones before deciding where they should reside. In order to handle this we have a new method `nova.compute.api.create_all_at_once()`. This method does things a little differently: 1. it validates all the fields passed into it. 2. it creates a single `reservation_id` for all of instances created. This is a UUID. 3. it creates a single `run_instance` request in the scheduler queue 4. a scheduler picks the message off the queue and works on it. -5. the scheduler sends off an OS API `POST /zones/select` command to each Child Zone. The `BODY` payload of the call contains the `request_spec`. -6. the Child Zones use the `request_spec` to compute a weighted list for each instance requested. No attempt to actually create an instance is done at this point. We're only estimating the suitability of the Zones. -7. if the Child Zone has its own Child Zone's, the `/zones/select` call will be sent down to them as well. +5. the scheduler sends off an OS API `POST /zones/select` command to each child Zone. The `BODY` payload of the call contains the `request_spec`. +6. the child Zones use the `request_spec` to compute a weighted list for each instance requested. No attempt to actually create an instance is done at this point. We're only estimating the suitability of the Zones. +7. if the child Zone has its own child Zones, the `/zones/select` call will be sent down to them as well. 8. Finally, when all the estimates have bubbled back to the Zone that initiated the call, all the results are merged, sorted and processed. -9. Now the instances can be created. The initiating Zone either forwards the `run_instance` message to the local Compute node to do the work, or it issues a `POST /servers` call to the relevant Child Zone. The parameters to the Child Zone call are the same as what was passed in by the user. +9. Now the instances can be created. The initiating Zone either forwards the `run_instance` message to the local Compute node to do the work, or it issues a `POST /servers` call to the relevant child Zone. The parameters to the child Zone call are the same as what was passed in by the user. 10. The `reservation_id` is passed back to the caller. Later we explain how the user can check on the status of the command with this `reservation_id`. The Catch ------------- -This all seems pretty straightforward but, like most things, there's a catch. Zones are expected to operate in complete isolation from each other. Each Zone has its own AMQP service, Database and set of Nova Services. But, for security reasons Zones should never leak information about the architectural layout internally. That means Zones cannot leak information about hostnames or service IP addresses outside of its world. +This all seems pretty straightforward but, like most things, there's a catch. Zones are expected to operate in complete isolation from each other. Each Zone has its own AMQP service, database and set of Nova services. But, for security reasons Zones should never leak information about the architectural layout internally. That means Zones cannot leak information about hostnames or service IP addresses outside of its world. -When `POST /zones/select` is called to estimate which Compute node to use, time passes until the `POST /servers` call is issued. If we only passed the Weight back from the `select` we would have to re-compute the appropriate Compute node for the create command ... and we could end up with a different host. Somehow we need to remember the results of our computations and pass them outside of the Zone. Now, we could store this information in the local database and return a reference to it, but remember that the vast majority of weights are going be ignored. Storing them in the database would result in a flood of disk access and then we have to clean up all these entries periodically. Recall that there are going to be many many `select` calls issued to Child Zones asking for estimates. +When `POST /zones/select` is called to estimate which compute node to use, time passes until the `POST /servers` call is issued. If we only passed the weight back from the `select` we would have to re-compute the appropriate compute node for the create command ... and we could end up with a different host. Somehow we need to remember the results of our computations and pass them outside of the Zone. Now, we could store this information in the local database and return a reference to it, but remember that the vast majority of weights are going be ignored. Storing them in the database would result in a flood of disk access and then we have to clean up all these entries periodically. Recall that there are going to be many many `select` calls issued to child Zones asking for estimates. -Instead, we take a rather innovative approach to the problem. We encrypt all the child zone internal details and pass them back the to parent Zone. If the parent zone decides to use a child Zone for the instance it simply passes the encrypted data back to the child during the `POST /servers` call as an extra parameter. The child Zone can then decrypt the hint and go directly to the Compute node previously selected. If the estimate isn't used, it is simply discarded by the parent. +Instead, we take a rather innovative approach to the problem. We encrypt all the child zone internal details and pass them back the to parent Zone. If the parent zone decides to use a child Zone for the instance it simply passes the encrypted data back to the child during the `POST /servers` call as an extra parameter. The child Zone can then decrypt the hint and go directly to the Compute node previously selected. If the estimate isn't used, it is simply discarded by the parent. It's for this reason that it is so important that each Zone defines a unique encryption key via `--build_plan_encryption_key` In the case of nested child Zones, each Zone re-encrypts the weighted list results and passes those values to the parent. Throughout the `nova.api.openstack.servers`, `nova.api.openstack.zones`, `nova.compute.api.create*` and `nova.scheduler.zone_aware_scheduler` code you'll see references to `blob` and `child_blob`. These are the encrypted hints about which Compute node to use. -Reservation ID's +Reservation IDs --------------- NOTE: The features described in this section are related to the up-coming 'merge-4' branch. The OpenStack API allows a user to list all the instances they own via the `GET /servers/` command or the details on a particular instance via `GET /servers/###`. This mechanism is usually sufficient since OS API only allows for creating one instance at a time, unlike the EC2 API which allows you to specify a quantity of instances to be created. -NOTE: currently the `GET /servers` command is not Zone-aware since all operations done in child Zones are done via a single administrative account. Therefore, asking a child Zone to `GET /servers` would return all the active instances ... and that would be bad. Later, when the Keystone Auth system is integrated with Nova, this functionality will be enabled. +NOTE: currently the `GET /servers` command is not Zone-aware since all operations done in child Zones are done via a single administrative account. Therefore, asking a child Zone to `GET /servers` would return all the active instances ... and that would be what the user intended. Later, when the Keystone Auth system is integrated with Nova, this functionality will be enabled. -We could use the OS API 1.1 Extensions mechanism to accept a `num_instances` parameter, but this would result in a different return code. Instead of getting back an `Instance` record, we would be getting back a `reservation_id`. So, instead, we've implemented a new command `POST /zones/servers` command which is nearly identical to `POST /servers` except that it takes a `num_instances` parameter and returns a `reservation_id`. Perhaps in OS API 2.x we can unify these approaches. +We could use the OS API 1.1 Extensions mechanism to accept a `num_instances` parameter, but this would result in a different return code. Instead of getting back an `Instance` record, we would be getting back a `reservation_id`. So, instead, we've implemented a new command `POST /zones/boot` command which is nearly identical to `POST /servers` except that it takes a `num_instances` parameter and returns a `reservation_id`. Perhaps in OS API 2.x we can unify these approaches. Finally, we need to give the user a way to get information on each of the instances created under this `reservation_id`. Fortunately, this is still possible with the existing `GET /servers` command, so long as we add a new optional `reservation_id` parameter. @@ -119,14 +123,15 @@ Finally, we need to give the user a way to get information on each of the instan Host Filter -------------- -As we mentioned earlier, filtering hosts is a very deployment specific process. Service Providers may have a different set of criteria for filtering Compute nodes than a University. To faciliate this the `nova.scheduler.host_filter` module supports a variety of filtering strategies as well as an easy means for plugging in your own algorithms. +As we mentioned earlier, filtering hosts is a very deployment-specific process. Service Providers may have a different set of criteria for filtering Compute nodes than a University. To faciliate this the `nova.scheduler.host_filter` module supports a variety of filtering strategies as well as an easy means for plugging in your own algorithms. + +The filter used is determined by the `--default_host_filter` flag, which points to a Python Class. By default this flag is set to `nova.scheduler.host_filter.AllHostsFilter` which simply returns all available hosts. But there are others: -The filter used is determined by the `--default_host_filter` flag, which points to a Python Class. By default this flag is set to `nova.scheduler.host_filter.AllHostsFilter` which simply returns all available hosts. But there are others. + * `nova.scheduler.host_filter.InstanceTypeFilter` provides host filtering based on the memory and disk size specified in the `InstanceType` record passed into `run_instance`. -`nova.scheduler.host_filter.InstanceTypeFilter` provides host filtering based on the memory and disk size specified in the `InstanceType` record passed into `run_instance`. -`nova.scheduler.host_filter.JSONFilter` filters hosts based on simple JSON expression grammar. Using a LISP-like JSON structure the caller can request instances based on criteria well beyond what `InstanceType` specifies. See `nova.tests.test_host_filter` for examples. + * `nova.scheduler.host_filter.JSONFilter` filters hosts based on simple JSON expression grammar. Using a LISP-like JSON structure the caller can request instances based on criteria well beyond what `InstanceType` specifies. See `nova.tests.test_host_filter` for examples. -To create your own `HostFilter` the user simply has to derive from `nova.scheduler.host_filter.HostFilter` and implement two methods: `instance_type_to_filter` and `filter_hosts`. Since Nova is currently dependent on the `InstanceType` structure, the `instance_type_to_filter` method should take an `InstanceType` and turn it into an internal data structure usable by your filter. This is for backward compatibility with existing OpenStack and EC2 API calls. If you decide the create your own call for creating instances not based on `Flavors` or `InstanceTypes` you can ignore this method. The real work is done in `filter_hosts` which must return a list of weight tuples for each appropriate host. The set of all available hosts is in the `ZoneManager` object passed into the call as well as the filter query. The weight tuple contains (``, ``) where `` is whatever you want it to be. +To create your own `HostFilter` the user simply has to derive from `nova.scheduler.host_filter.HostFilter` and implement two methods: `instance_type_to_filter` and `filter_hosts`. Since Nova is currently dependent on the `InstanceType` structure, the `instance_type_to_filter` method should take an `InstanceType` and turn it into an internal data structure usable by your filter. This is for backward compatibility with existing OpenStack and EC2 API calls. If you decide to create your own call for creating instances not based on `Flavors` or `InstanceTypes` you can ignore this method. The real work is done in `filter_hosts` which must return a list of host tuples for each appropriate host. The set of all available hosts is in the `ZoneManager` object passed into the call as well as the filter query. The host tuple contains (``, ``) where `` is whatever you want it to be. Cost Scheduler Weighing -------------- @@ -134,9 +139,9 @@ Every `ZoneAwareScheduler` derivation must also override the `weigh_hosts` metho Simple Zone Aware Scheduling -------------- -The easiest way to get started with the Zone Aware Scheduler is to use the `nova.scheduler.host_filter.HostFilterScheduler`. This scheduler uses the default Host Filter as and the `weight_hosts` method simply returns a weight of 1 for all hosts. But, from this, you can see calls being routed from Zone to Zone and follow the flow of things. +The easiest way to get started with the `ZoneAwareScheduler` is to use the `nova.scheduler.host_filter.HostFilterScheduler`. This scheduler uses the default Host Filter as and the `weight_hosts` method simply returns a weight of 1 for all hosts. But, from this, you can see calls being routed from Zone to Zone and follow the flow of things. -The `--scheduler_driver` flag is how you specify the Scheduler class name. +The `--scheduler_driver` flag is how you specify the scheduler class name. Flags -------------- @@ -153,9 +158,9 @@ All this Zone and Distributed Scheduler stuff can seem a little daunting to conf `--allow_admin_api` must be set for OS API to enable the new `/zones/*` commands. `--enable_zone_routing` must be set for OS API commands such as `create()`, `pause()` and `delete()` to get routed from Zone to Zone when looking for instances. -`--zone_name` is only required in Child Zones. The default Zone name is `nova`, but you may want to name your child Zones something useful. Duplicate Zone names are not an issue. +`--zone_name` is only required in child Zones. The default Zone name is `nova`, but you may want to name your child Zones something useful. Duplicate Zone names are not an issue. `build_plan_encryption_key` is the SHA-256 key for encrypting/decrypting the Host information when it leaves a Zone. Be sure to change this key for each Zone you create. Do not duplicate keys. -`scheduler_driver` is the real work horse of the operation. For Distributed Scheduler, you need to specify a class derived from `nova.scheduler.zone_aware_scheduler.ZoneAwareScheduler` +`scheduler_driver` is the real workhorse of the operation. For Distributed Scheduler, you need to specify a class derived from `nova.scheduler.zone_aware_scheduler.ZoneAwareScheduler`. `default_host_filter` is the host filter to be used for filtering candidate Compute nodes. Some optional flags which are handy for debugging are: -- cgit From 1eee07811f9fb5fd29192b17610a6b2d2e6c3578 Mon Sep 17 00:00:00 2001 From: William Wolf Date: Tue, 31 May 2011 13:34:33 -0400 Subject: added get_pagination_params function in common with tests, allow fake and local image services to accept filters, markers, and limits (but ignore them for now) --- nova/api/openstack/common.py | 31 ++++++++++++++++++++++ nova/api/openstack/images.py | 25 +++++++++++++++--- nova/image/fake.py | 4 +-- nova/image/glance.py | 10 ++++--- nova/image/local.py | 4 +-- nova/tests/api/openstack/fakes.py | 5 ++-- nova/tests/api/openstack/test_common.py | 46 +++++++++++++++++++++++++++++++++ nova/tests/image/test_glance.py | 2 +- 8 files changed, 113 insertions(+), 14 deletions(-) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 32cd689ca..69877cbce 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -36,6 +36,37 @@ XML_NS_V10 = 'http://docs.rackspacecloud.com/servers/api/v1.0' XML_NS_V11 = 'http://docs.openstack.org/compute/api/v1.1' +def get_pagination_params(request): + """ + Return marker, limit tuple from request + + @param request: `wsgi.Request` possibly containing 'marker' and 'limit' + GET variables. 'marker' is the id of the last element + the client has seen, and 'limit' is the maximum number + of items to return. If 'limit' is not specified, 0, or + > max_limit, we default to max_limit. Negative values + for either marker or limit will cause + exc.HTTPBadRequest() exceptions to be raised. + """ + try: + marker = int(request.GET.get('marker', 0)) + except ValueError: + raise webob.exc.HTTPBadRequest(_('offset param must be an integer')) + + try: + limit = int(request.GET.get('limit', 0)) + except ValueError: + raise webob.exc.HTTPBadRequest(_('limit param must be an integer')) + + if limit < 0: + raise webob.exc.HTTPBadRequest(_('limit param must be positive')) + + if marker < 0: + raise webob.exc.HTTPBadRequest(_('marker param must be positive')) + + return(marker, limit) + + def limited(items, request, max_limit=FLAGS.osapi_max_limit): """ Return a slice of items according to requested offset and limit. diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index c96b1c3e3..afe0f79de 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -74,7 +74,7 @@ class Controller(common.OpenstackController): """ context = req.environ['nova.context'] images = self._image_service.detail(context) - images = self._limited_items(images, req) + images = self._limit_items(images, req) builder = self.get_builder(req).build return dict(images=[builder(image, detail=True) for image in images]) @@ -157,5 +157,24 @@ class ControllerV11(Controller): def get_default_xmlns(self, req): return common.XML_NS_V11 - def _limit_items(self, items, req): - return common.limited_by_marker(items, req) + def index(self, req): + """Return an index listing of images available to the request. + + :param req: `wsgi.Request` object + """ + context = req.environ['nova.context'] + (marker, limit) = common.get_pagination_params(req) + images = self._image_service.index(context, marker, limit) + builder = self.get_builder(req).build + return dict(images=[builder(image, detail=False) for image in images]) + + def detail(self, req): + """Return a detailed index listing of images available to the request. + + :param req: `wsgi.Request` object. + """ + context = req.environ['nova.context'] + (marker, limit) = common.get_pagination_params(req) + images = self._image_service.detail(context, marker, limit) + builder = self.get_builder(req).build + return dict(images=[builder(image, detail=True) for image in images]) diff --git a/nova/image/fake.py b/nova/image/fake.py index b400b2adb..4aa4219fe 100644 --- a/nova/image/fake.py +++ b/nova/image/fake.py @@ -52,11 +52,11 @@ class FakeImageService(service.BaseImageService): self.create(None, image) super(FakeImageService, self).__init__() - def index(self, context): + def index(self, context, filters=None, marker=None, limit=None): """Returns list of images.""" return copy.deepcopy(self.images.values()) - def detail(self, context): + def detail(self, context, filters=None, marker=None, limit=None): """Return list of detailed image information.""" return copy.deepcopy(self.images.values()) diff --git a/nova/image/glance.py b/nova/image/glance.py index 193e37273..e084ed8ae 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -58,23 +58,25 @@ class GlanceImageService(service.BaseImageService): else: self.client = client - def index(self, context): + def index(self, context, marker=None, limit=None): """Calls out to Glance for a list of images available.""" # NOTE(sirp): We need to use `get_images_detailed` and not # `get_images` here because we need `is_public` and `properties` # included so we can filter by user filtered = [] - image_metas = self.client.get_images_detailed() + image_metas = self.client.get_images_detailed( + marker=marker, limit=limit) for image_meta in image_metas: if self._is_image_available(context, image_meta): meta_subset = utils.subset_dict(image_meta, ('id', 'name')) filtered.append(meta_subset) return filtered - def detail(self, context): + def detail(self, context, marker=None, limit=None): """Calls out to Glance for a list of detailed image information.""" filtered = [] - image_metas = self.client.get_images_detailed() + image_metas = self.client.get_images_detailed( + marker=marker, limit=limit) for image_meta in image_metas: if self._is_image_available(context, image_meta): base_image_meta = self._translate_to_base(image_meta) diff --git a/nova/image/local.py b/nova/image/local.py index 918180bae..f320cc60c 100644 --- a/nova/image/local.py +++ b/nova/image/local.py @@ -63,7 +63,7 @@ class LocalImageService(service.BaseImageService): images.append(unhexed_image_id) return images - def index(self, context): + def index(self, context, filters=None, marker=None, limit=None): filtered = [] image_metas = self.detail(context) for image_meta in image_metas: @@ -71,7 +71,7 @@ class LocalImageService(service.BaseImageService): filtered.append(meta) return filtered - def detail(self, context): + def detail(self, context, filters=None, marker=None, limit=None): images = [] for image_id in self._ids(): try: diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index bf51239e6..2e28e421c 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -166,11 +166,12 @@ def stub_out_glance(stubs, initial_fixtures=None): def __init__(self, initial_fixtures): self.fixtures = initial_fixtures or [] - def fake_get_images(self): + def fake_get_images(self, filters=None, marker=None, limit=None): return [dict(id=f['id'], name=f['name']) for f in self.fixtures] - def fake_get_images_detailed(self): + def fake_get_images_detailed(self, filters=None, + marker=None, limit=None): return copy.deepcopy(self.fixtures) def fake_get_image_meta(self, image_id): diff --git a/nova/tests/api/openstack/test_common.py b/nova/tests/api/openstack/test_common.py index 8f57c5b67..34597c7ac 100644 --- a/nova/tests/api/openstack/test_common.py +++ b/nova/tests/api/openstack/test_common.py @@ -25,6 +25,7 @@ from webob import Request from nova import test from nova.api.openstack.common import limited +from nova.api.openstack.common import get_pagination_params class LimiterTest(test.TestCase): @@ -169,3 +170,48 @@ class LimiterTest(test.TestCase): """ req = Request.blank('/?offset=-30') self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) + + +class PaginationParamsTest(test.TestCase): + """ + Unit tests for the `nova.api.openstack.common.get_pagination_params` + method which takes in a request object and returns 'marker' and 'limit' + GET params. + """ + + def test_no_params(self): + """ + Test no params. + """ + req = Request.blank('/') + self.assertEqual(get_pagination_params(req), (0, 0)) + + def test_valid_marker(self): + """ + Test valid marker param. + """ + req = Request.blank('/?marker=1') + self.assertEqual(get_pagination_params(req), (1, 0)) + + def test_invalid_marker(self): + """ + Test invalid marker param. + """ + req = Request.blank('/?marker=-2') + self.assertRaises(webob.exc.HTTPBadRequest, + get_pagination_params, req) + + def test_valid_limit(self): + """ + Test valid limit param. + """ + req = Request.blank('/?limit=10') + self.assertEqual(get_pagination_params(req), (0, 10)) + + def test_invalid_limit(self): + """ + Test invalid limit param. + """ + req = Request.blank('/?limit=-2') + self.assertRaises(webob.exc.HTTPBadRequest, + get_pagination_params, req) diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py index 109905ded..041da1e13 100644 --- a/nova/tests/image/test_glance.py +++ b/nova/tests/image/test_glance.py @@ -34,7 +34,7 @@ class StubGlanceClient(object): def get_image_meta(self, image_id): return self.images[image_id] - def get_images_detailed(self): + def get_images_detailed(self, filters=None, marker=None, limit=None): return self.images.itervalues() def get_image(self, image_id): -- cgit From f668339effa089360c1989082c83afc35489f71e Mon Sep 17 00:00:00 2001 From: William Wolf Date: Tue, 31 May 2011 14:21:15 -0400 Subject: added tests for GlanceImageService --- nova/tests/api/openstack/fakes.py | 38 ++++++++++++++++-- nova/tests/api/openstack/test_images.py | 68 +++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 3 deletions(-) diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 2e28e421c..e7006debe 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -167,12 +167,44 @@ def stub_out_glance(stubs, initial_fixtures=None): self.fixtures = initial_fixtures or [] def fake_get_images(self, filters=None, marker=None, limit=None): - return [dict(id=f['id'], name=f['name']) - for f in self.fixtures] + found = True + if marker: found = False + if limit == 0: limit = None + + fixtures = [] + count = 0 + for f in self.fixtures: + if limit and count >= limit: + break + if found: + fixtures.append(f) + count = count + 1 + if f['id'] == marker: + found = True + + return [dict(id=f['id'], name=f['name']) + for f in fixtures] def fake_get_images_detailed(self, filters=None, marker=None, limit=None): - return copy.deepcopy(self.fixtures) + found = True + if marker: found = False + if limit == 0: limit = None + + fixtures = [] + count = 0 + for f in self.fixtures: + if limit and count >= limit: + break + if found: + fixtures.append(f) + count = count + 1 + if f['id'] == marker: + found = True + + + return fixtures + def fake_get_image_meta(self, image_id): image = self._find_image(image_id) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 82bf66e49..310fbd5b4 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -226,6 +226,74 @@ class GlanceImageServiceTest(_BaseImageServiceTests): expected = {'name': 'test image', 'properties': {}} self.assertDictMatch(self.sent_to_glance['metadata'], expected) + def test_index_default_limit(self): + fixtures = [] + ids = [] + for i in range(10): + fixture = self._make_fixture('TestImage %d' % (i)) + fixtures.append(fixture) + ids.append(self.service.create(self.context, fixture)['id']) + + image_metas = self.service.index(self.context) + i = 0 + for meta in image_metas: + expected = {'id': 'DONTCARE', + 'name': 'TestImage %d' % (i)} + self.assertDictMatch(meta, expected) + i = i + 1 + + def test_index_marker(self): + fixtures = [] + ids = [] + for i in range(10): + fixture = self._make_fixture('TestImage %d' % (i)) + fixtures.append(fixture) + ids.append(self.service.create(self.context, fixture)['id']) + + image_metas = self.service.index(self.context, marker=ids[1]) + self.assertEquals(len(image_metas), 8) + i = 2 + for meta in image_metas: + expected = {'id': 'DONTCARE', + 'name': 'TestImage %d' % (i)} + self.assertDictMatch(meta, expected) + i = i + 1 + + def test_index_limit(self): + fixtures = [] + ids = [] + for i in range(10): + fixture = self._make_fixture('TestImage %d' % (i)) + fixtures.append(fixture) + ids.append(self.service.create(self.context, fixture)['id']) + + image_metas = self.service.index(self.context, limit=3) + self.assertEquals(len(image_metas), 3) + + def test_index_marker_and_limit(self): + fixtures = [] + ids = [] + for i in range(10): + fixture = self._make_fixture('TestImage %d' % (i)) + fixtures.append(fixture) + ids.append(self.service.create(self.context, fixture)['id']) + + image_metas = self.service.index(self.context, marker=ids[3], limit=1) + self.assertEquals(len(image_metas), 1) + i = 4 + for meta in image_metas: + expected = {'id': 'DONTCARE', + 'name': 'TestImage %d' % (i)} + self.assertDictMatch(meta, expected) + i = i + 1 + + def test_detail(self): + fixture = self._make_fixture('test image') + image_id = self.service.create(self.context, fixture)['id'] + image_metas = self.service.index(self.context) + expected = [{'id': 'DONTCARE', 'name': 'test image'}] + self.assertDictListMatch(image_metas, expected) + class ImageControllerWithGlanceServiceTest(test.TestCase): """ -- cgit From f16f55a08038c78200a490055183104fc6a9348d Mon Sep 17 00:00:00 2001 From: William Wolf Date: Tue, 31 May 2011 16:43:25 -0400 Subject: added tests for image detail requests --- nova/tests/api/openstack/test_images.py | 56 +++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 310fbd5b4..e8657683a 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -287,12 +287,56 @@ class GlanceImageServiceTest(_BaseImageServiceTests): self.assertDictMatch(meta, expected) i = i + 1 - def test_detail(self): - fixture = self._make_fixture('test image') - image_id = self.service.create(self.context, fixture)['id'] - image_metas = self.service.index(self.context) - expected = [{'id': 'DONTCARE', 'name': 'test image'}] - self.assertDictListMatch(image_metas, expected) + def test_detail_marker(self): + fixtures = [] + ids = [] + for i in range(10): + fixture = self._make_fixture('TestImage %d' % (i)) + fixtures.append(fixture) + ids.append(self.service.create(self.context, fixture)['id']) + + image_metas = self.service.detail(self.context, marker=ids[1]) + self.assertEquals(len(image_metas), 8) + i = 2 + for meta in image_metas: + expected = {'id': 'DONTCARE', 'status': None, + 'is_public': True, 'properties':{ + 'updated': None, 'created': None + }, + 'name': 'TestImage %d' % (i)} + self.assertDictMatch(meta, expected) + i = i + 1 + + def test_detail_limit(self): + fixtures = [] + ids = [] + for i in range(10): + fixture = self._make_fixture('TestImage %d' % (i)) + fixtures.append(fixture) + ids.append(self.service.create(self.context, fixture)['id']) + + image_metas = self.service.detail(self.context, limit=3) + self.assertEquals(len(image_metas), 3) + + def test_detail_marker_and_limit(self): + fixtures = [] + ids = [] + for i in range(10): + fixture = self._make_fixture('TestImage %d' % (i)) + fixtures.append(fixture) + ids.append(self.service.create(self.context, fixture)['id']) + + image_metas = self.service.detail(self.context, marker=ids[3], limit=3) + self.assertEquals(len(image_metas), 3) + i = 4 + for meta in image_metas: + expected = {'id': 'DONTCARE', 'status': None, + 'is_public': True, 'properties':{ + 'updated': None, 'created': None + }, + 'name': 'TestImage %d' % (i)} + self.assertDictMatch(meta, expected) + i = i + 1 class ImageControllerWithGlanceServiceTest(test.TestCase): -- cgit From b8f2f8d63608d76af41fd218dddb955bdc656354 Mon Sep 17 00:00:00 2001 From: William Wolf Date: Wed, 1 Jun 2011 10:00:15 -0400 Subject: fix filtering tests --- nova/api/openstack/images.py | 8 ++++++-- nova/image/glance.py | 4 ++-- nova/tests/api/openstack/test_images.py | 30 ++++++++++++++++++++---------- 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index 20e6f38ce..8afd38a4f 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -181,8 +181,10 @@ class ControllerV11(Controller): :param req: `wsgi.Request` object """ context = req.environ['nova.context'] + filters = self._get_filters(req) (marker, limit) = common.get_pagination_params(req) - images = self._image_service.index(context, marker, limit) + images = self._image_service.index( + context, filters=filters, marker=marker, limit=limit) builder = self.get_builder(req).build return dict(images=[builder(image, detail=False) for image in images]) @@ -192,7 +194,9 @@ class ControllerV11(Controller): :param req: `wsgi.Request` object. """ context = req.environ['nova.context'] + filters = self._get_filters(req) (marker, limit) = common.get_pagination_params(req) - images = self._image_service.detail(context, marker, limit) + images = self._image_service.detail( + context, filters=filters, marker=marker, limit=limit) builder = self.get_builder(req).build return dict(images=[builder(image, detail=True) for image in images]) diff --git a/nova/image/glance.py b/nova/image/glance.py index 09b2240ab..06f546027 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -58,7 +58,7 @@ class GlanceImageService(service.BaseImageService): else: self.client = client - def index(self, context, marker=None, limit=None, filters=None): + def index(self, context, filters=None, marker=None, limit=None): """Calls out to Glance for a list of images available.""" # NOTE(sirp): We need to use `get_images_detailed` and not # `get_images` here because we need `is_public` and `properties` @@ -73,7 +73,7 @@ class GlanceImageService(service.BaseImageService): filtered.append(meta_subset) return filtered - def detail(self, context, marker=None, limit=None, filters=None): + def detail(self, context, filters=None, marker=None, limit=None): """Calls out to Glance for a list of detailed image information.""" filtered = [] image_metas = self.client.get_images_detailed(marker=marker, diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index d6b01400e..c859a31de 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -826,7 +826,8 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): image_service = mocker.CreateMockAnything() context = object() filters = {'name': 'testname'} - image_service.index(context, filters).AndReturn([]) + image_service.index( + context, filters=filters, marker=0, limit=0).AndReturn([]) mocker.ReplayAll() request = webob.Request.blank( '/v1.1/images?name=testname') @@ -840,7 +841,8 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): image_service = mocker.CreateMockAnything() context = object() filters = {'status': 'ACTIVE'} - image_service.index(context, filters).AndReturn([]) + image_service.index( + context, filters=filters, marker=0, limit=0).AndReturn([]) mocker.ReplayAll() request = webob.Request.blank( '/v1.1/images?status=ACTIVE') @@ -854,7 +856,8 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): image_service = mocker.CreateMockAnything() context = object() filters = {'property-test': '3'} - image_service.index(context, filters).AndReturn([]) + image_service.index( + context, filters=filters, marker=0, limit=0).AndReturn([]) mocker.ReplayAll() request = webob.Request.blank( '/v1.1/images?property-test=3') @@ -868,7 +871,8 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): image_service = mocker.CreateMockAnything() context = object() filters = {'status': 'ACTIVE'} - image_service.index(context, filters).AndReturn([]) + image_service.index( + context, filters=filters, marker=0, limit=0).AndReturn([]) mocker.ReplayAll() request = webob.Request.blank( '/v1.1/images?status=ACTIVE&UNSUPPORTEDFILTER=testname') @@ -882,7 +886,8 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): image_service = mocker.CreateMockAnything() context = object() filters = {} - image_service.index(context, filters).AndReturn([]) + image_service.index( + context, filters=filters, marker=0, limit=0).AndReturn([]) mocker.ReplayAll() request = webob.Request.blank( '/v1.1/images') @@ -896,7 +901,8 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): image_service = mocker.CreateMockAnything() context = object() filters = {'name': 'testname'} - image_service.detail(context, filters).AndReturn([]) + image_service.detail( + context, filters=filters, marker=0, limit=0).AndReturn([]) mocker.ReplayAll() request = webob.Request.blank( '/v1.1/images/detail?name=testname') @@ -910,7 +916,8 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): image_service = mocker.CreateMockAnything() context = object() filters = {'status': 'ACTIVE'} - image_service.detail(context, filters).AndReturn([]) + image_service.detail( + context, filters=filters, marker=0, limit=0).AndReturn([]) mocker.ReplayAll() request = webob.Request.blank( '/v1.1/images/detail?status=ACTIVE') @@ -924,7 +931,8 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): image_service = mocker.CreateMockAnything() context = object() filters = {'property-test': '3'} - image_service.detail(context, filters).AndReturn([]) + image_service.detail( + context, filters=filters, marker=0, limit=0).AndReturn([]) mocker.ReplayAll() request = webob.Request.blank( '/v1.1/images/detail?property-test=3') @@ -938,7 +946,8 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): image_service = mocker.CreateMockAnything() context = object() filters = {'status': 'ACTIVE'} - image_service.detail(context, filters).AndReturn([]) + image_service.detail( + context, filters=filters, marker=0, limit=0).AndReturn([]) mocker.ReplayAll() request = webob.Request.blank( '/v1.1/images/detail?status=ACTIVE&UNSUPPORTEDFILTER=testname') @@ -952,7 +961,8 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): image_service = mocker.CreateMockAnything() context = object() filters = {} - image_service.detail(context, filters).AndReturn([]) + image_service.detail( + context, filters=filters, marker=0, limit=0).AndReturn([]) mocker.ReplayAll() request = webob.Request.blank( '/v1.1/images/detail') -- cgit From 3fa4ece45eea12f0923c55d87130c668bafd2751 Mon Sep 17 00:00:00 2001 From: William Wolf Date: Wed, 1 Jun 2011 10:31:53 -0400 Subject: fix pep8 issues --- nova/api/openstack/common.py | 4 ++-- nova/tests/api/openstack/fakes.py | 13 ++++++++----- nova/tests/api/openstack/test_common.py | 6 +++--- nova/tests/api/openstack/test_images.py | 16 +++++++--------- nova/tests/api/openstack/test_servers.py | 3 +-- nova/tests/integrated/test_servers.py | 9 +++------ 6 files changed, 24 insertions(+), 27 deletions(-) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 69877cbce..b0d368dfa 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -39,13 +39,13 @@ XML_NS_V11 = 'http://docs.openstack.org/compute/api/v1.1' def get_pagination_params(request): """ Return marker, limit tuple from request - + @param request: `wsgi.Request` possibly containing 'marker' and 'limit' GET variables. 'marker' is the id of the last element the client has seen, and 'limit' is the maximum number of items to return. If 'limit' is not specified, 0, or > max_limit, we default to max_limit. Negative values - for either marker or limit will cause + for either marker or limit will cause exc.HTTPBadRequest() exceptions to be raised. """ try: diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 6395280fd..bc21d66b4 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -168,8 +168,10 @@ def stub_out_glance(stubs, initial_fixtures=None): def fake_get_images(self, filters=None, marker=None, limit=None): found = True - if marker: found = False - if limit == 0: limit = None + if marker: + found = False + if limit == 0: + limit = None fixtures = [] count = 0 @@ -188,8 +190,10 @@ def stub_out_glance(stubs, initial_fixtures=None): def fake_get_images_detailed(self, filters=None, marker=None, limit=None): found = True - if marker: found = False - if limit == 0: limit = None + if marker: + found = False + if limit == 0: + limit = None fixtures = [] count = 0 @@ -202,7 +206,6 @@ def stub_out_glance(stubs, initial_fixtures=None): if f['id'] == marker: found = True - return fixtures def fake_get_image_meta(self, image_id): diff --git a/nova/tests/api/openstack/test_common.py b/nova/tests/api/openstack/test_common.py index 34597c7ac..55142ffe1 100644 --- a/nova/tests/api/openstack/test_common.py +++ b/nova/tests/api/openstack/test_common.py @@ -174,8 +174,8 @@ class LimiterTest(test.TestCase): class PaginationParamsTest(test.TestCase): """ - Unit tests for the `nova.api.openstack.common.get_pagination_params` - method which takes in a request object and returns 'marker' and 'limit' + Unit tests for the `nova.api.openstack.common.get_pagination_params` + method which takes in a request object and returns 'marker' and 'limit' GET params. """ @@ -185,7 +185,7 @@ class PaginationParamsTest(test.TestCase): """ req = Request.blank('/') self.assertEqual(get_pagination_params(req), (0, 0)) - + def test_valid_marker(self): """ Test valid marker param. diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index c859a31de..667f2866b 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -238,7 +238,7 @@ class GlanceImageServiceTest(_BaseImageServiceTests): image_metas = self.service.index(self.context) i = 0 for meta in image_metas: - expected = {'id': 'DONTCARE', + expected = {'id': 'DONTCARE', 'name': 'TestImage %d' % (i)} self.assertDictMatch(meta, expected) i = i + 1 @@ -255,7 +255,7 @@ class GlanceImageServiceTest(_BaseImageServiceTests): self.assertEquals(len(image_metas), 8) i = 2 for meta in image_metas: - expected = {'id': 'DONTCARE', + expected = {'id': 'DONTCARE', 'name': 'TestImage %d' % (i)} self.assertDictMatch(meta, expected) i = i + 1 @@ -283,7 +283,7 @@ class GlanceImageServiceTest(_BaseImageServiceTests): self.assertEquals(len(image_metas), 1) i = 4 for meta in image_metas: - expected = {'id': 'DONTCARE', + expected = {'id': 'DONTCARE', 'name': 'TestImage %d' % (i)} self.assertDictMatch(meta, expected) i = i + 1 @@ -301,9 +301,8 @@ class GlanceImageServiceTest(_BaseImageServiceTests): i = 2 for meta in image_metas: expected = {'id': 'DONTCARE', 'status': None, - 'is_public': True, 'properties':{ - 'updated': None, 'created': None - }, + 'is_public': True, 'properties': { + 'updated': None, 'created': None}, 'name': 'TestImage %d' % (i)} self.assertDictMatch(meta, expected) i = i + 1 @@ -332,9 +331,8 @@ class GlanceImageServiceTest(_BaseImageServiceTests): i = 4 for meta in image_metas: expected = {'id': 'DONTCARE', 'status': None, - 'is_public': True, 'properties':{ - 'updated': None, 'created': None - }, + 'is_public': True, 'properties': { + 'updated': None, 'created': None}, 'name': 'TestImage %d' % (i)} self.assertDictMatch(meta, expected) i = i + 1 diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index fbde5c9ce..20379e2bd 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -774,8 +774,7 @@ class ServersTest(test.TestCase): def server_update(context, id, params): filtered_dict = dict( - display_name='server_test' - ) + display_name='server_test') self.assertEqual(params, filtered_dict) return filtered_dict diff --git a/nova/tests/integrated/test_servers.py b/nova/tests/integrated/test_servers.py index a67fa1bb5..35c6bb34f 100644 --- a/nova/tests/integrated/test_servers.py +++ b/nova/tests/integrated/test_servers.py @@ -194,8 +194,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase): post = {} post['rebuild'] = { "imageRef": "https://localhost/v1.1/32278/images/2", - "name": "blah" - } + "name": "blah"} self.api.post_server_action(created_server_id, post) LOG.debug("rebuilt server: %s" % created_server) @@ -224,8 +223,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase): post = {} post['rebuild'] = { "imageRef": "https://localhost/v1.1/32278/images/2", - "name": "blah" - } + "name": "blah"} metadata = {} for i in range(30): @@ -267,8 +265,7 @@ class ServersTest(integrated_helpers._IntegratedTestBase): post = {} post['rebuild'] = { "imageRef": "https://localhost/v1.1/32278/images/2", - "name": "blah" - } + "name": "blah"} metadata = {} post['rebuild']['metadata'] = metadata -- cgit From 15257606e5346f0bf9a67145e5d4df7ba57c386a Mon Sep 17 00:00:00 2001 From: William Wolf Date: Wed, 1 Jun 2011 14:58:17 -0400 Subject: touch ups --- nova/image/glance.py | 12 ++++++------ nova/tests/api/openstack/test_common.py | 3 +-- nova/tests/integrated/test_servers.py | 9 ++++++--- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/nova/image/glance.py b/nova/image/glance.py index 06f546027..61308431d 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -64,9 +64,9 @@ class GlanceImageService(service.BaseImageService): # `get_images` here because we need `is_public` and `properties` # included so we can filter by user filtered = [] - image_metas = self.client.get_images_detailed(marker=marker, - limit=limit, - filters=filters) + image_metas = self.client.get_images_detailed(filters=filters, + marker=marker, + limit=limit) for image_meta in image_metas: if self._is_image_available(context, image_meta): meta_subset = utils.subset_dict(image_meta, ('id', 'name')) @@ -76,9 +76,9 @@ class GlanceImageService(service.BaseImageService): def detail(self, context, filters=None, marker=None, limit=None): """Calls out to Glance for a list of detailed image information.""" filtered = [] - image_metas = self.client.get_images_detailed(marker=marker, - limit=limit, - filters=filters) + image_metas = self.client.get_images_detailed(filters=filters, + marker=marker, + limit=limit) for image_meta in image_metas: if self._is_image_available(context, image_meta): base_image_meta = self._translate_to_base(image_meta) diff --git a/nova/tests/api/openstack/test_common.py b/nova/tests/api/openstack/test_common.py index 55142ffe1..c4a6e3ebf 100644 --- a/nova/tests/api/openstack/test_common.py +++ b/nova/tests/api/openstack/test_common.py @@ -24,8 +24,7 @@ import webob.exc from webob import Request from nova import test -from nova.api.openstack.common import limited -from nova.api.openstack.common import get_pagination_params +from nova.api.openstack.common import limited, get_pagination_params class LimiterTest(test.TestCase): diff --git a/nova/tests/integrated/test_servers.py b/nova/tests/integrated/test_servers.py index 35c6bb34f..fcb517cf5 100644 --- a/nova/tests/integrated/test_servers.py +++ b/nova/tests/integrated/test_servers.py @@ -194,7 +194,8 @@ class ServersTest(integrated_helpers._IntegratedTestBase): post = {} post['rebuild'] = { "imageRef": "https://localhost/v1.1/32278/images/2", - "name": "blah"} + "name": "blah", + } self.api.post_server_action(created_server_id, post) LOG.debug("rebuilt server: %s" % created_server) @@ -223,7 +224,8 @@ class ServersTest(integrated_helpers._IntegratedTestBase): post = {} post['rebuild'] = { "imageRef": "https://localhost/v1.1/32278/images/2", - "name": "blah"} + "name": "blah", + } metadata = {} for i in range(30): @@ -265,7 +267,8 @@ class ServersTest(integrated_helpers._IntegratedTestBase): post = {} post['rebuild'] = { "imageRef": "https://localhost/v1.1/32278/images/2", - "name": "blah"} + "name": "blah", + } metadata = {} post['rebuild']['metadata'] = metadata -- cgit From 5d89721f5fa3212146749236c666f0e584c8590f Mon Sep 17 00:00:00 2001 From: John Tran Date: Wed, 1 Jun 2011 16:27:51 -0700 Subject: merged, with trunk, fixed the test failure, and split the test into 3 as per peer review. --- nova/tests/test_cloud.py | 45 ++++++++++++++++++++++++--------------------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 04fd02ba3..02b7c8a38 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -445,38 +445,41 @@ class CloudTestCase(test.TestCase): self.cloud.delete_key_pair(self.context, 'test') def test_run_instances(self): - all_instances = db.instance_get_all(context.get_admin_context()) - self.assertEqual(0, len(all_instances)) - - def fake_show_decrypt(self, context, id): - return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, - 'type': 'machine', 'image_state': 'decrypting'}} - - def fake_show_no_state(self, context, id): - return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, - 'type': 'machine'}} - - image_id = FLAGS.default_image - instance_type = FLAGS.default_instance_type - max_count = 1 - kwargs = {'image_id': image_id, - 'instance_type': instance_type, - 'max_count': max_count} + kwargs = {'image_id': FLAGS.default_image, + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1} run_instances = self.cloud.run_instances - # when image has valid image_state result = run_instances(self.context, **kwargs) instance = result['instancesSet'][0] self.assertEqual(instance['imageId'], 'ami-00000001') self.assertEqual(instance['displayName'], 'Server 1') self.assertEqual(instance['instanceId'], 'i-00000001') - self.assertEqual(instance['instanceState']['name'], 'scheduling') + self.assertEqual(instance['instanceState']['name'], 'networking') self.assertEqual(instance['instanceType'], 'm1.small') - # when image doesn't have 'image_state' attr at all + + def test_run_instances_image_state_none(self): + kwargs = {'image_id': FLAGS.default_image, + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1} + run_instances = self.cloud.run_instances + def fake_show_no_state(self, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, + 'type': 'machine'}} + self.stubs.UnsetAll() self.stubs.Set(local.LocalImageService, 'show', fake_show_no_state) self.assertRaises(exception.ApiError, run_instances, self.context, **kwargs) - # when image has 'image_state' yet not 'available' + + def test_run_instances_image_state_invalid(self): + kwargs = {'image_id': FLAGS.default_image, + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1} + run_instances = self.cloud.run_instances + def fake_show_decrypt(self, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, + 'type': 'machine', 'image_state': 'decrypting'}} + self.stubs.UnsetAll() self.stubs.Set(local.LocalImageService, 'show', fake_show_decrypt) self.assertRaises(exception.ApiError, run_instances, -- cgit From 4fb46873ef4332c6570d3ac5559557745056dee6 Mon Sep 17 00:00:00 2001 From: William Wolf Date: Wed, 1 Jun 2011 23:09:37 -0400 Subject: cleanup based on waldon's comments, also caught a few other issues --- nova/api/openstack/common.py | 18 +--- nova/api/openstack/images.py | 8 +- nova/api/openstack/servers.py | 1 + nova/tests/api/openstack/fakes.py | 25 ++--- nova/tests/api/openstack/test_common.py | 180 +++++++++++++------------------ nova/tests/api/openstack/test_images.py | 33 ++++-- nova/tests/api/openstack/test_servers.py | 1 + nova/tests/integrated/test_servers.py | 3 + 8 files changed, 115 insertions(+), 154 deletions(-) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 342cc8b2e..c9e3dbb64 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -50,7 +50,7 @@ def get_pagination_params(request): try: marker = int(request.GET.get('marker', 0)) except ValueError: - raise webob.exc.HTTPBadRequest(_('offset param must be an integer')) + raise webob.exc.HTTPBadRequest(_('marker param must be an integer')) try: limit = int(request.GET.get('limit', 0)) @@ -102,19 +102,11 @@ def limited(items, request, max_limit=FLAGS.osapi_max_limit): def limited_by_marker(items, request, max_limit=FLAGS.osapi_max_limit): """Return a slice of items according to the requested marker and limit.""" + print "TEST LIMIT" + (marker, limit) = get_pagination_params(request) - try: - marker = int(request.GET.get('marker', 0)) - except ValueError: - raise webob.exc.HTTPBadRequest(_('marker param must be an integer')) - - try: - limit = int(request.GET.get('limit', max_limit)) - except ValueError: - raise webob.exc.HTTPBadRequest(_('limit param must be an integer')) - - if limit < 0: - raise webob.exc.HTTPBadRequest(_('limit param must be positive')) + if limit == 0: + limit = max_limit limit = min(max_limit, limit) start_index = 0 diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index dcedd3db2..4ef9a5974 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -46,9 +46,6 @@ class Controller(object): self._compute_service = compute_service or compute.API() self._image_service = image_service or _default_service - def _limit_items(self, items, req): - return common.limited(items, req) - def index(self, req): """Return an index listing of images available to the request. @@ -162,13 +159,11 @@ class ControllerV11(Controller): base_url = request.application_url return images_view.ViewBuilderV11(base_url) - def get_default_xmlns(self, req): - return common.XML_NS_V11 - def index(self, req): """Return an index listing of images available to the request. :param req: `wsgi.Request` object + """ context = req.environ['nova.context'] filters = self._get_filters(req) @@ -182,6 +177,7 @@ class ControllerV11(Controller): """Return a detailed index listing of images available to the request. :param req: `wsgi.Request` object. + """ context = req.environ['nova.context'] filters = self._get_filters(req) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index f2ce64e78..ad556ca84 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -55,6 +55,7 @@ class Controller(object): def detail(self, req): """ Returns a list of server details for a given user """ + print "DETAIL" return self._items(req, is_detail=True) def _image_id_from_req_data(self, data): diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index bc21d66b4..67cd395ad 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -166,7 +166,7 @@ def stub_out_glance(stubs, initial_fixtures=None): def __init__(self, initial_fixtures): self.fixtures = initial_fixtures or [] - def fake_get_images(self, filters=None, marker=None, limit=None): + def _filter_images(self, filters=None, marker=None, limit=None): found = True if marker: found = False @@ -184,29 +184,16 @@ def stub_out_glance(stubs, initial_fixtures=None): if f['id'] == marker: found = True + return fixtures + + def fake_get_images(self, filters=None, marker=None, limit=None): + fixtures = self._filter_images(filters, marker, limit) return [dict(id=f['id'], name=f['name']) for f in fixtures] def fake_get_images_detailed(self, filters=None, marker=None, limit=None): - found = True - if marker: - found = False - if limit == 0: - limit = None - - fixtures = [] - count = 0 - for f in self.fixtures: - if limit and count >= limit: - break - if found: - fixtures.append(f) - count = count + 1 - if f['id'] == marker: - found = True - - return fixtures + return self._filter_images(filters, marker, limit) def fake_get_image_meta(self, image_id): image = self._find_image(image_id) diff --git a/nova/tests/api/openstack/test_common.py b/nova/tests/api/openstack/test_common.py index c4a6e3ebf..9a9d9125c 100644 --- a/nova/tests/api/openstack/test_common.py +++ b/nova/tests/api/openstack/test_common.py @@ -24,7 +24,7 @@ import webob.exc from webob import Request from nova import test -from nova.api.openstack.common import limited, get_pagination_params +from nova.api.openstack import common class LimiterTest(test.TestCase): @@ -35,9 +35,7 @@ class LimiterTest(test.TestCase): """ def setUp(self): - """ - Run before each test. - """ + """ Run before each test. """ super(LimiterTest, self).setUp() self.tiny = range(1) self.small = range(10) @@ -45,130 +43,112 @@ class LimiterTest(test.TestCase): self.large = range(10000) def test_limiter_offset_zero(self): - """ - Test offset key works with 0. - """ + """ Test offset key works with 0. """ req = Request.blank('/?offset=0') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium) - self.assertEqual(limited(self.large, req), self.large[:1000]) + self.assertEqual(common.limited(self.tiny, req), self.tiny) + self.assertEqual(common.limited(self.small, req), self.small) + self.assertEqual(common.limited(self.medium, req), self.medium) + self.assertEqual(common.limited(self.large, req), self.large[:1000]) def test_limiter_offset_medium(self): - """ - Test offset key works with a medium sized number. - """ + """ Test offset key works with a medium sized number. """ req = Request.blank('/?offset=10') - self.assertEqual(limited(self.tiny, req), []) - self.assertEqual(limited(self.small, req), self.small[10:]) - self.assertEqual(limited(self.medium, req), self.medium[10:]) - self.assertEqual(limited(self.large, req), self.large[10:1010]) + self.assertEqual(common.limited(self.tiny, req), []) + self.assertEqual(common.limited(self.small, req), self.small[10:]) + self.assertEqual(common.limited(self.medium, req), self.medium[10:]) + self.assertEqual(common.limited(self.large, req), self.large[10:1010]) def test_limiter_offset_over_max(self): - """ - Test offset key works with a number over 1000 (max_limit). - """ + """ Test offset key works with a number over 1000 (max_limit). """ req = Request.blank('/?offset=1001') - self.assertEqual(limited(self.tiny, req), []) - self.assertEqual(limited(self.small, req), []) - self.assertEqual(limited(self.medium, req), []) - self.assertEqual(limited(self.large, req), self.large[1001:2001]) + self.assertEqual(common.limited(self.tiny, req), []) + self.assertEqual(common.limited(self.small, req), []) + self.assertEqual(common.limited(self.medium, req), []) + self.assertEqual( + common.limited(self.large, req), self.large[1001:2001]) def test_limiter_offset_blank(self): - """ - Test offset key works with a blank offset. - """ + """ Test offset key works with a blank offset. """ req = Request.blank('/?offset=') - self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) + self.assertRaises( + webob.exc.HTTPBadRequest, common.limited, self.tiny, req) def test_limiter_offset_bad(self): - """ - Test offset key works with a BAD offset. - """ + """ Test offset key works with a BAD offset. """ req = Request.blank(u'/?offset=\u0020aa') - self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) + self.assertRaises( + webob.exc.HTTPBadRequest, common.limited, self.tiny, req) def test_limiter_nothing(self): - """ - Test request with no offset or limit - """ + """ Test request with no offset or limit """ req = Request.blank('/') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium) - self.assertEqual(limited(self.large, req), self.large[:1000]) + self.assertEqual(common.limited(self.tiny, req), self.tiny) + self.assertEqual(common.limited(self.small, req), self.small) + self.assertEqual(common.limited(self.medium, req), self.medium) + self.assertEqual(common.limited(self.large, req), self.large[:1000]) def test_limiter_limit_zero(self): - """ - Test limit of zero. - """ + """ Test limit of zero. """ req = Request.blank('/?limit=0') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium) - self.assertEqual(limited(self.large, req), self.large[:1000]) + self.assertEqual(common.limited(self.tiny, req), self.tiny) + self.assertEqual(common.limited(self.small, req), self.small) + self.assertEqual(common.limited(self.medium, req), self.medium) + self.assertEqual(common.limited(self.large, req), self.large[:1000]) def test_limiter_limit_medium(self): - """ - Test limit of 10. - """ + """ Test limit of 10. """ req = Request.blank('/?limit=10') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium[:10]) - self.assertEqual(limited(self.large, req), self.large[:10]) + self.assertEqual(common.limited(self.tiny, req), self.tiny) + self.assertEqual(common.limited(self.small, req), self.small) + self.assertEqual(common.limited(self.medium, req), self.medium[:10]) + self.assertEqual(common.limited(self.large, req), self.large[:10]) def test_limiter_limit_over_max(self): - """ - Test limit of 3000. - """ + """ Test limit of 3000. """ req = Request.blank('/?limit=3000') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium) - self.assertEqual(limited(self.large, req), self.large[:1000]) + self.assertEqual(common.limited(self.tiny, req), self.tiny) + self.assertEqual(common.limited(self.small, req), self.small) + self.assertEqual(common.limited(self.medium, req), self.medium) + self.assertEqual(common.limited(self.large, req), self.large[:1000]) def test_limiter_limit_and_offset(self): - """ - Test request with both limit and offset. - """ + """ Test request with both limit and offset. """ items = range(2000) req = Request.blank('/?offset=1&limit=3') - self.assertEqual(limited(items, req), items[1:4]) + self.assertEqual(common.limited(items, req), items[1:4]) req = Request.blank('/?offset=3&limit=0') - self.assertEqual(limited(items, req), items[3:1003]) + self.assertEqual(common.limited(items, req), items[3:1003]) req = Request.blank('/?offset=3&limit=1500') - self.assertEqual(limited(items, req), items[3:1003]) + self.assertEqual(common.limited(items, req), items[3:1003]) req = Request.blank('/?offset=3000&limit=10') - self.assertEqual(limited(items, req), []) + self.assertEqual(common.limited(items, req), []) def test_limiter_custom_max_limit(self): - """ - Test a max_limit other than 1000. - """ + """ Test a max_limit other than 1000. """ items = range(2000) req = Request.blank('/?offset=1&limit=3') - self.assertEqual(limited(items, req, max_limit=2000), items[1:4]) + self.assertEqual( + common.limited(items, req, max_limit=2000), items[1:4]) req = Request.blank('/?offset=3&limit=0') - self.assertEqual(limited(items, req, max_limit=2000), items[3:]) + self.assertEqual( + common.limited(items, req, max_limit=2000), items[3:]) req = Request.blank('/?offset=3&limit=2500') - self.assertEqual(limited(items, req, max_limit=2000), items[3:]) + self.assertEqual( + common.limited(items, req, max_limit=2000), items[3:]) req = Request.blank('/?offset=3000&limit=10') - self.assertEqual(limited(items, req, max_limit=2000), []) + self.assertEqual(common.limited(items, req, max_limit=2000), []) def test_limiter_negative_limit(self): - """ - Test a negative limit. - """ + """ Test a negative limit. """ req = Request.blank('/?limit=-3000') - self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) + self.assertRaises( + webob.exc.HTTPBadRequest, common.limited, self.tiny, req) def test_limiter_negative_offset(self): - """ - Test a negative offset. - """ + """ Test a negative offset. """ req = Request.blank('/?offset=-30') - self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) + self.assertRaises( + webob.exc.HTTPBadRequest, common.limited, self.tiny, req) class PaginationParamsTest(test.TestCase): @@ -179,38 +159,28 @@ class PaginationParamsTest(test.TestCase): """ def test_no_params(self): - """ - Test no params. - """ + """ Test no params. """ req = Request.blank('/') - self.assertEqual(get_pagination_params(req), (0, 0)) + self.assertEqual(common.get_pagination_params(req), (0, 0)) def test_valid_marker(self): - """ - Test valid marker param. - """ + """ Test valid marker param. """ req = Request.blank('/?marker=1') - self.assertEqual(get_pagination_params(req), (1, 0)) + self.assertEqual(common.get_pagination_params(req), (1, 0)) def test_invalid_marker(self): - """ - Test invalid marker param. - """ + """ Test invalid marker param. """ req = Request.blank('/?marker=-2') - self.assertRaises(webob.exc.HTTPBadRequest, - get_pagination_params, req) + self.assertRaises( + webob.exc.HTTPBadRequest, common.get_pagination_params, req) def test_valid_limit(self): - """ - Test valid limit param. - """ + """ Test valid limit param. """ req = Request.blank('/?limit=10') - self.assertEqual(get_pagination_params(req), (0, 10)) + self.assertEqual(common.get_pagination_params(req), (0, 10)) def test_invalid_limit(self): - """ - Test invalid limit param. - """ + """ Test invalid limit param. """ req = Request.blank('/?limit=-2') - self.assertRaises(webob.exc.HTTPBadRequest, - get_pagination_params, req) + self.assertRaises( + webob.exc.HTTPBadRequest, common.get_pagination_params, req) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 667f2866b..38823c377 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -239,7 +239,7 @@ class GlanceImageServiceTest(_BaseImageServiceTests): i = 0 for meta in image_metas: expected = {'id': 'DONTCARE', - 'name': 'TestImage %d' % (i)} + 'name': 'TestImage %d' % (i)} self.assertDictMatch(meta, expected) i = i + 1 @@ -256,7 +256,7 @@ class GlanceImageServiceTest(_BaseImageServiceTests): i = 2 for meta in image_metas: expected = {'id': 'DONTCARE', - 'name': 'TestImage %d' % (i)} + 'name': 'TestImage %d' % (i)} self.assertDictMatch(meta, expected) i = i + 1 @@ -284,7 +284,7 @@ class GlanceImageServiceTest(_BaseImageServiceTests): i = 4 for meta in image_metas: expected = {'id': 'DONTCARE', - 'name': 'TestImage %d' % (i)} + 'name': 'TestImage %d' % (i)} self.assertDictMatch(meta, expected) i = i + 1 @@ -300,10 +300,17 @@ class GlanceImageServiceTest(_BaseImageServiceTests): self.assertEquals(len(image_metas), 8) i = 2 for meta in image_metas: - expected = {'id': 'DONTCARE', 'status': None, - 'is_public': True, 'properties': { - 'updated': None, 'created': None}, - 'name': 'TestImage %d' % (i)} + expected = { + 'id': 'DONTCARE', + 'status': None, + 'is_public': True, + 'name': 'TestImage %d' % (i), + 'properties': { + 'updated': None, + 'created': None, + }, + } + self.assertDictMatch(meta, expected) i = i + 1 @@ -330,10 +337,14 @@ class GlanceImageServiceTest(_BaseImageServiceTests): self.assertEquals(len(image_metas), 3) i = 4 for meta in image_metas: - expected = {'id': 'DONTCARE', 'status': None, - 'is_public': True, 'properties': { - 'updated': None, 'created': None}, - 'name': 'TestImage %d' % (i)} + expected = { + 'id': 'DONTCARE', + 'status': None, + 'is_public': True, + 'name': 'TestImage %d' % (i), + 'properties': { + 'updated': None, 'created': None}, + } self.assertDictMatch(meta, expected) i = i + 1 diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 51def1980..3de7865cd 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -448,6 +448,7 @@ class ServersTest(test.TestCase): req = webob.Request.blank('/v1.1/servers?limit=2&marker=asdf') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 400) + print "BODY",res.body self.assertTrue(res.body.find('marker param') > -1) def _setup_for_create_instance(self): diff --git a/nova/tests/integrated/test_servers.py b/nova/tests/integrated/test_servers.py index fcb517cf5..89987b645 100644 --- a/nova/tests/integrated/test_servers.py +++ b/nova/tests/integrated/test_servers.py @@ -88,12 +88,15 @@ class ServersTest(integrated_helpers._IntegratedTestBase): # Check it's there found_server = self.api.get_server(created_server_id) + print "FOUND_SERVER:", found_server self.assertEqual(created_server_id, found_server['id']) # It should also be in the all-servers list servers = self.api.get_servers() + print "SERVERS:", servers server_ids = [server['id'] for server in servers] self.assertTrue(created_server_id in server_ids) + return # Wait (briefly) for creation retries = 0 -- cgit From 5ded1f2c1d0d14b3c04df137f7cc6a0b65e53fda Mon Sep 17 00:00:00 2001 From: William Wolf Date: Wed, 1 Jun 2011 23:11:50 -0400 Subject: got rid of print debugs --- nova/api/openstack/common.py | 1 - nova/api/openstack/servers.py | 1 - nova/tests/api/openstack/test_images.py | 2 +- nova/tests/api/openstack/test_servers.py | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index c9e3dbb64..559b44ef5 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -102,7 +102,6 @@ def limited(items, request, max_limit=FLAGS.osapi_max_limit): def limited_by_marker(items, request, max_limit=FLAGS.osapi_max_limit): """Return a slice of items according to the requested marker and limit.""" - print "TEST LIMIT" (marker, limit) = get_pagination_params(request) if limit == 0: diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index ad556ca84..f2ce64e78 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -55,7 +55,6 @@ class Controller(object): def detail(self, req): """ Returns a list of server details for a given user """ - print "DETAIL" return self._items(req, is_detail=True) def _image_id_from_req_data(self, data): diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 38823c377..c2b03c281 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -306,7 +306,7 @@ class GlanceImageServiceTest(_BaseImageServiceTests): 'is_public': True, 'name': 'TestImage %d' % (i), 'properties': { - 'updated': None, + 'updated': None, 'created': None, }, } diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 3de7865cd..51def1980 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -448,7 +448,6 @@ class ServersTest(test.TestCase): req = webob.Request.blank('/v1.1/servers?limit=2&marker=asdf') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 400) - print "BODY",res.body self.assertTrue(res.body.find('marker param') > -1) def _setup_for_create_instance(self): -- cgit From 7b24750057cfef1d0f14b21cb83b1ac9c0869836 Mon Sep 17 00:00:00 2001 From: William Wolf Date: Thu, 2 Jun 2011 08:53:13 -0400 Subject: got rid of prints --- nova/tests/integrated/test_servers.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/tests/integrated/test_servers.py b/nova/tests/integrated/test_servers.py index 89987b645..1733896fd 100644 --- a/nova/tests/integrated/test_servers.py +++ b/nova/tests/integrated/test_servers.py @@ -88,12 +88,10 @@ class ServersTest(integrated_helpers._IntegratedTestBase): # Check it's there found_server = self.api.get_server(created_server_id) - print "FOUND_SERVER:", found_server self.assertEqual(created_server_id, found_server['id']) # It should also be in the all-servers list servers = self.api.get_servers() - print "SERVERS:", servers server_ids = [server['id'] for server in servers] self.assertTrue(created_server_id in server_ids) return -- cgit From e28a6e96ec45439ed24a363f27d0421d720add0b Mon Sep 17 00:00:00 2001 From: William Wolf Date: Thu, 2 Jun 2011 09:34:01 -0400 Subject: move index and detail functions to v10 controller --- nova/api/openstack/images.py | 48 ++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index 4ef9a5974..7f06c53df 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -46,30 +46,6 @@ class Controller(object): self._compute_service = compute_service or compute.API() self._image_service = image_service or _default_service - def index(self, req): - """Return an index listing of images available to the request. - - :param req: `wsgi.Request` object - """ - context = req.environ['nova.context'] - filters = self._get_filters(req) - images = self._image_service.index(context, filters) - images = common.limited(images, req) - builder = self.get_builder(req).build - return dict(images=[builder(image, detail=False) for image in images]) - - def detail(self, req): - """Return a detailed index listing of images available to the request. - - :param req: `wsgi.Request` object. - """ - context = req.environ['nova.context'] - filters = self._get_filters(req) - images = self._image_service.detail(context, filters) - images = common.limited(images, req) - builder = self.get_builder(req).build - return dict(images=[builder(image, detail=True) for image in images]) - def _get_filters(self, req): """ Return a dictionary of query param filters from the request @@ -150,6 +126,30 @@ class ControllerV10(Controller): base_url = request.application_url return images_view.ViewBuilderV10(base_url) + def index(self, req): + """Return an index listing of images available to the request. + + :param req: `wsgi.Request` object + """ + context = req.environ['nova.context'] + filters = self._get_filters(req) + images = self._image_service.index(context, filters) + images = common.limited(images, req) + builder = self.get_builder(req).build + return dict(images=[builder(image, detail=False) for image in images]) + + def detail(self, req): + """Return a detailed index listing of images available to the request. + + :param req: `wsgi.Request` object. + """ + context = req.environ['nova.context'] + filters = self._get_filters(req) + images = self._image_service.detail(context, filters) + images = common.limited(images, req) + builder = self.get_builder(req).build + return dict(images=[builder(image, detail=True) for image in images]) + class ControllerV11(Controller): """Version 1.1 specific controller logic.""" -- cgit From 9f1027069c47ea83e1dfca9bed48b2a403463689 Mon Sep 17 00:00:00 2001 From: William Wolf Date: Thu, 2 Jun 2011 11:58:17 -0400 Subject: got rid of more test debugging stuff that shouldnt have made it in --- nova/tests/integrated/test_servers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/tests/integrated/test_servers.py b/nova/tests/integrated/test_servers.py index 1733896fd..fcb517cf5 100644 --- a/nova/tests/integrated/test_servers.py +++ b/nova/tests/integrated/test_servers.py @@ -94,7 +94,6 @@ class ServersTest(integrated_helpers._IntegratedTestBase): servers = self.api.get_servers() server_ids = [server['id'] for server in servers] self.assertTrue(created_server_id in server_ids) - return # Wait (briefly) for creation retries = 0 -- cgit From 7ca707c1cbfb3164d4b6f706a4e9720e54bcc35f Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Thu, 2 Jun 2011 12:02:16 -0400 Subject: Minor comment formatting changes. --- nova/api/openstack/common.py | 6 +++--- nova/api/openstack/images.py | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 559b44ef5..40fb59765 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -36,16 +36,16 @@ XML_NS_V11 = 'http://docs.openstack.org/compute/api/v1.1' def get_pagination_params(request): - """ - Return marker, limit tuple from request + """Return marker, limit tuple from request. - @param request: `wsgi.Request` possibly containing 'marker' and 'limit' + :param request: `wsgi.Request` possibly containing 'marker' and 'limit' GET variables. 'marker' is the id of the last element the client has seen, and 'limit' is the maximum number of items to return. If 'limit' is not specified, 0, or > max_limit, we default to max_limit. Negative values for either marker or limit will cause exc.HTTPBadRequest() exceptions to be raised. + """ try: marker = int(request.GET.get('marker', 0)) diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index 7f06c53df..73249b485 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -130,6 +130,7 @@ class ControllerV10(Controller): """Return an index listing of images available to the request. :param req: `wsgi.Request` object + """ context = req.environ['nova.context'] filters = self._get_filters(req) @@ -142,6 +143,7 @@ class ControllerV10(Controller): """Return a detailed index listing of images available to the request. :param req: `wsgi.Request` object. + """ context = req.environ['nova.context'] filters = self._get_filters(req) -- cgit From 29eec21f6752ef2c03412213a74aa12745286c82 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Fri, 3 Jun 2011 05:23:43 -0700 Subject: little tweaks --- doc/source/devref/distributed_scheduler.rst | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/doc/source/devref/distributed_scheduler.rst b/doc/source/devref/distributed_scheduler.rst index 28ba20af7..eb6a1a03e 100644 --- a/doc/source/devref/distributed_scheduler.rst +++ b/doc/source/devref/distributed_scheduler.rst @@ -1,7 +1,3 @@ - - - - .. Copyright 2011 OpenStack LLC All Rights Reserved. @@ -40,7 +36,7 @@ Some Costs are more esoteric. Consider a rule that says we should prefer Hosts t An example of some other costs might include selecting: * a GPU-based host over a standard CPU * a host with fast ethernet over a 10mbps line -* a host than can run Windows instances +* a host that can run Windows instances * a host in the EU vs North America * etc -- cgit From 8739529368cb755d33c3d8c532dd1c5d86f0bf85 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Fri, 3 Jun 2011 08:50:30 -0400 Subject: Implement OSAPI v1.1 style image create. --- nova/api/openstack/images.py | 11 ++++++++- nova/tests/api/openstack/fakes.py | 2 +- nova/tests/api/openstack/test_images.py | 41 +++++++++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 2 deletions(-) diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index 59d9e3082..48ea04248 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -123,7 +123,7 @@ class Controller(object): raise webob.exc.HTTPBadRequest() try: - server_id = body["image"]["serverId"] + server_id = self._server_id_from_req_data(body) image_name = body["image"]["name"] except KeyError: raise webob.exc.HTTPBadRequest() @@ -135,6 +135,9 @@ class Controller(object): """Indicates that you must use a Controller subclass.""" raise NotImplementedError + def _server_id_from_req_data(self, data): + raise NotImplementedError() + class ControllerV10(Controller): """Version 1.0 specific controller logic.""" @@ -144,6 +147,9 @@ class ControllerV10(Controller): base_url = request.application_url return images_view.ViewBuilderV10(base_url) + def _server_id_from_req_data(self, data): + return data['image']['serverId'] + class ControllerV11(Controller): """Version 1.1 specific controller logic.""" @@ -153,6 +159,9 @@ class ControllerV11(Controller): base_url = request.application_url return images_view.ViewBuilderV11(base_url) + def _server_id_from_req_data(self, data): + return data['image']['serverRef'] + def create_resource(version='1.0'): controller = { diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 17d6d591c..e9b46f933 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -143,7 +143,7 @@ def stub_out_networking(stubs): def stub_out_compute_api_snapshot(stubs): def snapshot(self, context, instance_id, name): - return 123 + return dict(id='123') stubs.Set(nova.compute.API, 'snapshot', snapshot) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 9f1f28611..961c271ca 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -249,6 +249,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): fakes.stub_out_key_pair_funcs(self.stubs) self.fixtures = self._make_image_fixtures() fakes.stub_out_glance(self.stubs, initial_fixtures=self.fixtures) + fakes.stub_out_compute_api_snapshot(self.stubs) def tearDown(self): """Run after each test.""" @@ -871,6 +872,46 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 404) + def test_create_image(self): + + body = dict(image=dict(serverId='123', name='Backup 1')) + req = webob.Request.blank('/v1.0/images') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + response = req.get_response(fakes.wsgi_app()) + self.assertEqual(200, response.status_int) + + def test_create_image_no_server_id(self): + + body = dict(image=dict(name='Backup 1')) + req = webob.Request.blank('/v1.0/images') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + response = req.get_response(fakes.wsgi_app()) + self.assertEqual(400, response.status_int) + + def test_create_image_v1_1(self): + + body = dict(image=dict(serverRef='123', name='Backup 1')) + req = webob.Request.blank('/v1.1/images') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + response = req.get_response(fakes.wsgi_app()) + self.assertEqual(200, response.status_int) + + def test_create_image_v1_1_no_server_ref(self): + + body = dict(image=dict(name='Backup 1')) + req = webob.Request.blank('/v1.1/images') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + response = req.get_response(fakes.wsgi_app()) + self.assertEqual(400, response.status_int) + @classmethod def _make_image_fixtures(cls): image_id = 123 -- cgit From a9f21962a9e1e703730fbfae120129618b7a79ca Mon Sep 17 00:00:00 2001 From: Alex Meade Date: Fri, 3 Jun 2011 09:24:46 -0400 Subject: Fixed pylint: no metadata member in models.py --- nova/db/sqlalchemy/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index edb7ffe4b..82b521e77 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -47,6 +47,7 @@ class NovaBase(object): updated_at = Column(DateTime, onupdate=datetime.datetime.utcnow) deleted_at = Column(DateTime) deleted = Column(Boolean, default=False) + metadata = None def save(self, session=None): """Save this object.""" -- cgit From 0ef4a127e9539f90ac1d2f2846832ecc48b51e05 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Fri, 3 Jun 2011 09:31:43 -0400 Subject: Add serverRef to image metadata serialization list. --- nova/api/openstack/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index 48ea04248..1fa3267dc 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -177,7 +177,7 @@ def create_resource(version='1.0'): metadata = { "attributes": { "image": ["id", "name", "updated", "created", "status", - "serverId", "progress"], + "serverId", "progress", "serverRef"], "link": ["rel", "type", "href"], }, } -- cgit From b45d07ded9db7c92e03cea1427413d4dda95d869 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Fri, 3 Jun 2011 10:23:38 -0400 Subject: Make libvirt snapshotting work with images that don't have an 'architecture' property. --- nova/virt/libvirt/connection.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/virt/libvirt/connection.py b/nova/virt/libvirt/connection.py index c491418ae..98cdff311 100644 --- a/nova/virt/libvirt/connection.py +++ b/nova/virt/libvirt/connection.py @@ -403,8 +403,7 @@ class LibvirtConnection(driver.ComputeDriver): 'is_public': False, 'status': 'active', 'name': snapshot['name'], - 'properties': {'architecture': - base['properties']['architecture'], + 'properties': { 'kernel_id': instance['kernel_id'], 'image_location': 'snapshot', 'image_state': 'available', @@ -412,6 +411,9 @@ class LibvirtConnection(driver.ComputeDriver): 'ramdisk_id': instance['ramdisk_id'], } } + if 'architecture' in base['properties']: + arch = base['properties']['architecture'] + metadata['properties']['architecture'] = arch # Make the snapshot snapshot_name = uuid.uuid4().hex -- cgit From 5b00ca3ac874d0fff1eb2835cd4219f49d8a169f Mon Sep 17 00:00:00 2001 From: Alex Meade Date: Fri, 3 Jun 2011 11:08:43 -0400 Subject: Set pylint to ignore correct lines that it could not determine were correct, due to the means by which eventlet.green imported subprocess Minimized the number of these lines to ignore --- nova/utils.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 361fc9873..4e1b7c26a 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -142,24 +142,26 @@ def execute(*cmd, **kwargs): env = os.environ.copy() if addl_env: env.update(addl_env) + _PIPE = subprocess.PIPE #pylint: disable=E1101 obj = subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdin=_PIPE, + stdout=_PIPE, + stderr=_PIPE, env=env) result = None if process_input is not None: result = obj.communicate(process_input) else: result = obj.communicate() - obj.stdin.close() - if obj.returncode: - LOG.debug(_('Result was %s') % obj.returncode) + obj.stdin.close() #pylint: disable=E1101 + _returncode = obj.returncode #pylint: disable=E1101 + if _returncode: + LOG.debug(_('Result was %s') % _returncode) if type(check_exit_code) == types.IntType \ - and obj.returncode != check_exit_code: + and _returncode != check_exit_code: (stdout, stderr) = result raise exception.ProcessExecutionError( - exit_code=obj.returncode, + exit_code=_returncode, stdout=stdout, stderr=stderr, cmd=' '.join(cmd)) -- cgit From 24a90512f20310007f4ca8ab01da8e19a6b5bf6f Mon Sep 17 00:00:00 2001 From: Alex Meade Date: Fri, 3 Jun 2011 11:28:49 -0400 Subject: Removed unused and erroneous (yes, it was both) function --- nova/api/ec2/admin.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/nova/api/ec2/admin.py b/nova/api/ec2/admin.py index ea94d9c1f..4d981f70b 100644 --- a/nova/api/ec2/admin.py +++ b/nova/api/ec2/admin.py @@ -325,7 +325,3 @@ class AdminController(object): rv.append(host_dict(host, compute, instances, volume, volumes, now)) return {'hosts': rv} - - def describe_host(self, _context, name, **_kwargs): - """Returns status info for single node.""" - return host_dict(db.host_get(name)) -- cgit From eadabab8b70bdc4789615844e2263cbed7aa283c Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Fri, 3 Jun 2011 11:34:49 -0400 Subject: Added a test case for XML serialization. --- nova/tests/api/openstack/fakes.py | 3 ++- nova/tests/api/openstack/test_images.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index e9b46f933..601c1e9e4 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -143,7 +143,8 @@ def stub_out_networking(stubs): def stub_out_compute_api_snapshot(stubs): def snapshot(self, context, instance_id, name): - return dict(id='123') + return dict(id='123', status='ACTIVE', + properties=dict(instance_id='123')) stubs.Set(nova.compute.API, 'snapshot', snapshot) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 961c271ca..ae7025146 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -902,6 +902,39 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): response = req.get_response(fakes.wsgi_app()) self.assertEqual(200, response.status_int) + def test_create_image_v1_1_xml_serialization(self): + + body = dict(image=dict(serverRef='123', name='Backup 1')) + req = webob.Request.blank('/v1.1/images') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + req.headers["accept"] = "application/xml" + response = req.get_response(fakes.wsgi_app()) + self.assertEqual(200, response.status_int) + resp_xml = minidom.parseString(response.body.replace(" ", "")) + expected_href = "http://localhost/v1.1/images/123" + expected_image = minidom.parseString(""" + + + + + + + + """.replace(" ", "") % (locals())) + + self.assertEqual(expected_image.toxml(), resp_xml.toxml()) + def test_create_image_v1_1_no_server_ref(self): body = dict(image=dict(name='Backup 1')) -- cgit From 25c8e9318c1ffbf2f2c88d3ed644df9e81b92b04 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Fri, 3 Jun 2011 11:52:20 -0400 Subject: Fixed pip-requires double requirement. --- tools/pip-requires | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/pip-requires b/tools/pip-requires index 035e4347d..e81ef944a 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -34,4 +34,3 @@ coverage nosexcover GitPython paramiko -nova_adminclient -- cgit From f521426039e8a9cc5dccc2c7e7e1797cfe778d7e Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Fri, 3 Jun 2011 14:14:28 -0400 Subject: Updated to use the '/v1/images' URL when uploading images to glance in the Xen glance plugin. Fixes issue where snapshots failed to get uploaded. --- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 0c00d168b..46031ebe8 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -244,7 +244,7 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, os_type): conn = httplib.HTTPConnection(glance_host, glance_port) # NOTE(sirp): httplib under python2.4 won't accept a file-like object # to request - conn.putrequest('PUT', '/images/%s' % image_id) + conn.putrequest('PUT', '/v1/images/%s' % image_id) # NOTE(sirp): There is some confusion around OVF. Here's a summary of # where we currently stand: -- cgit From f6aa513024e14975709ef8facf1db6535eefbc44 Mon Sep 17 00:00:00 2001 From: Justin Shepherd Date: Fri, 3 Jun 2011 13:20:34 -0500 Subject: added 'nova-manage config list' which will list out all of the flags and their values. I also alphabetized the list of available categories --- bin/nova-manage | 33 ++++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 5de4d9e81..fb3810779 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -1081,24 +1081,35 @@ class ImageCommands(object): self._convert_images(machine_images) +class ConfigCommands(object): + """Class for exposing the flags defined by flag_file(s).""" + + def __init__(self): + pass + + def list(self): + print FLAGS.FlagsIntoString() + + CATEGORIES = [ - ('user', UserCommands), ('account', AccountCommands), - ('project', ProjectCommands), - ('role', RoleCommands), - ('shell', ShellCommands), - ('vpn', VpnCommands), + ('config', ConfigCommands), + ('db', DbCommands), ('fixed', FixedIpCommands), + ('flavor', InstanceTypeCommands), ('floating', FloatingIpCommands), + ('instance_type', InstanceTypeCommands), + ('image', ImageCommands), ('network', NetworkCommands), - ('vm', VmCommands), + ('project', ProjectCommands), + ('role', RoleCommands), ('service', ServiceCommands), - ('db', DbCommands), + ('shell', ShellCommands), + ('user', UserCommands), + ('version', VersionCommands), + ('vm', VmCommands), ('volume', VolumeCommands), - ('instance_type', InstanceTypeCommands), - ('image', ImageCommands), - ('flavor', InstanceTypeCommands), - ('version', VersionCommands)] + ('vpn', VpnCommands)] def lazy_match(name, key_value_tuples): -- cgit From 9c38da46d121e65707346473e6d51da3a2cf021f Mon Sep 17 00:00:00 2001 From: Alex Meade Date: Mon, 6 Jun 2011 09:18:13 -0400 Subject: Fixed incorrect exception --- nova/db/sqlalchemy/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index cf1a84cd5..6970a2168 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -738,7 +738,7 @@ def fixed_ip_get_all_by_instance(context, instance_id): filter_by(instance_id=instance_id).\ filter_by(deleted=False) if not rv: - raise exception.NoFloatingIpsFoundForInstance(instance_id=instance_id) + raise exception.NoFixedIpsFoundForInstance(instance_id=instance_id) return rv -- cgit From ec5e5bcd3592dca44d1d71455ccd99e2c7f24d26 Mon Sep 17 00:00:00 2001 From: Alex Meade Date: Mon, 6 Jun 2011 10:49:29 -0400 Subject: Small pylint fixes --- nova/api/openstack/extensions.py | 6 ++++-- nova/api/openstack/views/limits.py | 9 --------- nova/tests/xenapi/stubs.py | 4 ++-- 3 files changed, 6 insertions(+), 13 deletions(-) diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index 881b61733..9dad2f48d 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -137,7 +137,8 @@ class ActionExtensionResource(wsgi.Resource): def __init__(self, application): controller = ActionExtensionController(application) - super(ActionExtensionResource, self).__init__(controller) + #super(ActionExtensionResource, self).__init__(controller) + wsgi.Resource.__init__(self, controller) def add_action(self, action_name, handler): self.controller.add_action(action_name, handler) @@ -164,7 +165,8 @@ class RequestExtensionResource(wsgi.Resource): def __init__(self, application): controller = RequestExtensionController(application) - super(RequestExtensionResource, self).__init__(controller) + #super(RequestExtensionResource, self).__init__(controller) + wsgi.Resource.__init__(self, controller) def add_handler(self, handler): self.controller.add_handler(handler) diff --git a/nova/api/openstack/views/limits.py b/nova/api/openstack/views/limits.py index e21c9f2fd..934b4921a 100644 --- a/nova/api/openstack/views/limits.py +++ b/nova/api/openstack/views/limits.py @@ -29,9 +29,6 @@ class ViewBuilder(object): def _build_rate_limit(self, rate_limit): raise NotImplementedError() - def _build_absolute_limits(self, absolute_limit): - raise NotImplementedError() - def build(self, rate_limits, absolute_limits): rate_limits = self._build_rate_limits(rate_limits) absolute_limits = self._build_absolute_limits(absolute_limits) @@ -67,12 +64,6 @@ class ViewBuilder(object): limits[name] = value return limits - def _build_rate_limits(self, rate_limits): - raise NotImplementedError() - - def _build_rate_limit(self, rate_limit): - raise NotImplementedError() - class ViewBuilderV10(ViewBuilder): """Openstack API v1.0 limits view builder.""" diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 35308d95f..5d2d1641a 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -251,10 +251,10 @@ class FakeSessionForMigrationTests(fake.SessionBase): def __init__(self, uri): super(FakeSessionForMigrationTests, self).__init__(uri) - def VDI_get_by_uuid(*args): + def VDI_get_by_uuid(self, *args): return 'hurr' - def VDI_resize_online(*args): + def VDI_resize_online(self, *args): pass def VM_start(self, _1, ref, _2, _3): -- cgit From 3fb0b8fd8e4ad5911c85fddcb6ef5127fa4cd384 Mon Sep 17 00:00:00 2001 From: Alex Meade Date: Mon, 6 Jun 2011 11:00:51 -0400 Subject: Removed extraneous code --- nova/tests/xenapi/stubs.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 5d2d1641a..151a3e909 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -42,20 +42,6 @@ def stubout_instance_snapshot(stubs): stubs.Set(vm_utils.VMHelper, 'fetch_image', fake_fetch_image) - def fake_wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref, - original_parent_uuid): - from nova.virt.xenapi.fake import create_vdi - name_label = "instance-%s" % instance_id - #TODO: create fake SR record - sr_ref = "fakesr" - vdi_ref = create_vdi(name_label=name_label, read_only=False, - sr_ref=sr_ref, sharable=False) - vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref) - vdi_uuid = vdi_rec['uuid'] - return vdi_uuid - - stubs.Set(vm_utils.VMHelper, 'fetch_image', fake_fetch_image) - def fake_parse_xmlrpc_value(val): return val -- cgit From a2f74c2f706bdf45ec36348468b1ba5797fcde87 Mon Sep 17 00:00:00 2001 From: Alex Meade Date: Mon, 6 Jun 2011 11:20:25 -0400 Subject: Use super on an old style class --- nova/twistd.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/twistd.py b/nova/twistd.py index c07ed991f..15cf67825 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -78,7 +78,7 @@ def WrapTwistedOptions(wrapped): self._absorbParameters() self._absorbHandlers() - super(TwistedOptionsToFlags, self).__init__() + wrapped.__init__(self) def _absorbFlags(self): twistd_flags = [] @@ -163,12 +163,12 @@ def WrapTwistedOptions(wrapped): def parseArgs(self, *args): # TODO(termie): figure out a decent way of dealing with args #return - super(TwistedOptionsToFlags, self).parseArgs(*args) + wrapped.parseArgs(self, *args) def postOptions(self): self._doHandlers() - super(TwistedOptionsToFlags, self).postOptions() + wrapped.postOptions(self) def __getitem__(self, key): key = key.replace('-', '_') -- cgit From 0eb6db6f994963d519f9fe07e3dbc41e0c8079c6 Mon Sep 17 00:00:00 2001 From: Alex Meade Date: Mon, 6 Jun 2011 11:29:05 -0400 Subject: Removed Duplicate method --- nova/virt/xenapi/fake.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/nova/virt/xenapi/fake.py b/nova/virt/xenapi/fake.py index 76988b172..5d3b67417 100644 --- a/nova/virt/xenapi/fake.py +++ b/nova/virt/xenapi/fake.py @@ -340,10 +340,6 @@ class SessionBase(object): return db_ref['xenstore_data'][key] = None - def network_get_all_records_where(self, _1, _2): - # TODO (salvatore-orlando): filter table on _2 - return _db_content['network'] - def VM_add_to_xenstore_data(self, _1, vm_ref, key, value): db_ref = _db_content['VM'][vm_ref] if not 'xenstore_data' in db_ref: @@ -354,7 +350,7 @@ class SessionBase(object): #Always return 12GB available return 12 * 1024 * 1024 * 1024 - def host_call_plugin(*args): + def host_call_plugin(self, *args): return 'herp' def network_get_all_records_where(self, _1, filter): -- cgit From 3d481e551ac81a35cafcd79c2b17d2bd9c8a050f Mon Sep 17 00:00:00 2001 From: Alex Meade Date: Mon, 6 Jun 2011 11:39:34 -0400 Subject: Ignore complaining about dynamic definition --- nova/api/direct.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/direct.py b/nova/api/direct.py index ea20042a7..ea7425e19 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -324,7 +324,7 @@ class Limited(object): def __init__(self, proxy): self._proxy = proxy - if not self.__doc__: + if not self.__doc__: #pylint: disable=E0203 self.__doc__ = proxy.__doc__ if not self._allowed: self._allowed = [] -- cgit From 267178748e712098af4e55872029c5883af9a51c Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Mon, 6 Jun 2011 12:42:27 -0400 Subject: Change to a more generic error and update documentation. --- nova/db/sqlalchemy/api.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 56739e9db..6dbf53a6c 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -60,9 +60,7 @@ def is_user_context(context): def authorize_project_context(context, project_id): - """Ensures that the request context has permission to access the - given project. - """ + """Ensures a request has permission to access the given project.""" if is_user_context(context): if not context.project: raise exception.NotAuthorized() @@ -71,9 +69,7 @@ def authorize_project_context(context, project_id): def authorize_user_context(context, user_id): - """Ensures that the request context has permission to access the - given user. - """ + """Ensures a request has permission to access the given user.""" if is_user_context(context): if not context.user: raise exception.NotAuthorized() @@ -89,9 +85,12 @@ def can_read_deleted(context): def require_admin_context(f): - """Decorator used to indicate that the method requires an - administrator context. + """Decorator to require admin request context. + + The first argument to the wrapped function must be the context. + """ + def wrapper(*args, **kwargs): if not is_admin_context(args[0]): raise exception.AdminRequired() @@ -100,12 +99,19 @@ def require_admin_context(f): def require_context(f): - """Decorator used to indicate that the method requires either - an administrator or normal user context. + """Decorator to require *any* user or admin context. + + This does no authorization for user or project access matching, see + :py:func:`authorize_project_context` and + :py:func:`authorize_user_context`. + + The first argument to the wrapped function must be the context. + """ + def wrapper(*args, **kwargs): if not is_admin_context(args[0]) and not is_user_context(args[0]): - raise exception.AdminRequired() + raise exception.NotAuthorized() return f(*args, **kwargs) return wrapper -- cgit From 9fca0b2156f1e7f3d007916ef18b2ed9fbc761df Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Mon, 6 Jun 2011 15:59:20 -0400 Subject: Added test case for snapshoting base image without architecture. --- nova/tests/test_libvirt.py | 92 +++++++++++++++++++++++++++++++++------------- 1 file changed, 66 insertions(+), 26 deletions(-) diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index b6b36745a..d0bdaa738 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import copy import eventlet import mox import os @@ -125,6 +126,7 @@ class CacheConcurrencyTestCase(test.TestCase): class LibvirtConnTestCase(test.TestCase): + def setUp(self): super(LibvirtConnTestCase, self).setUp() connection._late_load_cheetah() @@ -207,6 +209,31 @@ class LibvirtConnTestCase(test.TestCase): self.mox.StubOutWithMock(connection.LibvirtConnection, '_conn') connection.LibvirtConnection._conn = fake + def fake_lookup(self, instance_name): + class FakeVirtDomain(object): + + def __init__(self): + pass + + def snapshotCreateXML(self, *args): + return None + + def XMLDesc(self, *args): + return """ + + + + + + + + """ + + return FakeVirtDomain() + + def fake_execute(self, *args): + open(args[-1], "a").close() + def create_service(self, **kwargs): service_ref = {'host': kwargs.get('host', 'dummy'), 'binary': 'nova-compute', @@ -283,43 +310,56 @@ class LibvirtConnTestCase(test.TestCase): self._check_xml_and_container(instance_data) def test_snapshot(self): + if not self.lazy_load_library_exists(): + return + FLAGS.image_service = 'nova.image.fake.FakeImageService' - # Only file-based instance storages are supported at the moment - test_xml = """ - - - - - - - - """ + # Start test + image_service = utils.import_object(FLAGS.image_service) - class FakeVirtDomain(object): + # Assuming that base image already exists in image_service + instance_ref = db.instance_create(self.context, self.test_instance) + properties = {'instance_id': instance_ref['id'], + 'user_id': str(self.context.user_id)} + snapshot_name = 'test-snap' + sent_meta = {'name': snapshot_name, 'is_public': False, + 'status': 'creating', 'properties': properties} + # Create new image. It will be updated in snapshot method + # To work with it from snapshot, the single image_service is needed + recv_meta = image_service.create(context, sent_meta) - def __init__(self): - pass + self.mox.StubOutWithMock(connection.LibvirtConnection, '_conn') + connection.LibvirtConnection._conn.lookupByName = self.fake_lookup + self.mox.StubOutWithMock(connection.utils, 'execute') + connection.utils.execute = self.fake_execute - def snapshotCreateXML(self, *args): - return None + self.mox.ReplayAll() - def XMLDesc(self, *args): - return test_xml + conn = connection.LibvirtConnection(False) + conn.snapshot(instance_ref, recv_meta['id']) - def fake_lookup(instance_name): - if instance_name == instance_ref.name: - return FakeVirtDomain() + snapshot = image_service.show(context, recv_meta['id']) + self.assertEquals(snapshot['properties']['image_state'], 'available') + self.assertEquals(snapshot['status'], 'active') + self.assertEquals(snapshot['name'], snapshot_name) - def fake_execute(*args): - # Touch filename to pass 'with open(out_path)' - open(args[-1], "a").close() + def test_snapshot_no_image_architecture(self): + if not self.lazy_load_library_exists(): + return + + FLAGS.image_service = 'nova.image.fake.FakeImageService' # Start test image_service = utils.import_object(FLAGS.image_service) + # Assign image_ref = 2 from nova/images/fakes for testing different + # base image + test_instance = copy.deepcopy(self.test_instance) + test_instance["image_ref"] = "2" + # Assuming that base image already exists in image_service - instance_ref = db.instance_create(self.context, self.test_instance) + instance_ref = db.instance_create(self.context, test_instance) properties = {'instance_id': instance_ref['id'], 'user_id': str(self.context.user_id)} snapshot_name = 'test-snap' @@ -330,9 +370,9 @@ class LibvirtConnTestCase(test.TestCase): recv_meta = image_service.create(context, sent_meta) self.mox.StubOutWithMock(connection.LibvirtConnection, '_conn') - connection.LibvirtConnection._conn.lookupByName = fake_lookup + connection.LibvirtConnection._conn.lookupByName = self.fake_lookup self.mox.StubOutWithMock(connection.utils, 'execute') - connection.utils.execute = fake_execute + connection.utils.execute = self.fake_execute self.mox.ReplayAll() -- cgit From 57df676a3302f8d754ef54e415d2fd82a4291f49 Mon Sep 17 00:00:00 2001 From: Alex Meade Date: Mon, 6 Jun 2011 15:59:39 -0400 Subject: Removed commented code --- nova/api/openstack/extensions.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index 9dad2f48d..54e17e23d 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -137,7 +137,6 @@ class ActionExtensionResource(wsgi.Resource): def __init__(self, application): controller = ActionExtensionController(application) - #super(ActionExtensionResource, self).__init__(controller) wsgi.Resource.__init__(self, controller) def add_action(self, action_name, handler): @@ -165,7 +164,6 @@ class RequestExtensionResource(wsgi.Resource): def __init__(self, application): controller = RequestExtensionController(application) - #super(RequestExtensionResource, self).__init__(controller) wsgi.Resource.__init__(self, controller) def add_handler(self, handler): -- cgit From e745c21724e5990874a12c4abff53127755185ea Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Mon, 6 Jun 2011 16:08:58 -0400 Subject: Use True/False instead of 1/0 when setting updating 'deleted' column attributes.Fixes casting issues when running nova with Postgres. --- nova/db/sqlalchemy/api.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 6dbf53a6c..103668b94 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1127,7 +1127,7 @@ def key_pair_destroy_all_by_user(context, user_id): with session.begin(): session.query(models.KeyPair).\ filter_by(user_id=user_id).\ - update({'deleted': 1, + update({'deleted': True, 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) @@ -1659,7 +1659,7 @@ def volume_destroy(context, volume_id): with session.begin(): session.query(models.Volume).\ filter_by(id=volume_id).\ - update({'deleted': 1, + update({'deleted': True, 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) session.query(models.ExportDevice).\ @@ -1817,7 +1817,7 @@ def snapshot_destroy(context, snapshot_id): with session.begin(): session.query(models.Snapshot).\ filter_by(id=snapshot_id).\ - update({'deleted': 1, + update({'deleted': True, 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) @@ -1972,17 +1972,17 @@ def security_group_destroy(context, security_group_id): with session.begin(): session.query(models.SecurityGroup).\ filter_by(id=security_group_id).\ - update({'deleted': 1, + update({'deleted': True, 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupInstanceAssociation).\ filter_by(security_group_id=security_group_id).\ - update({'deleted': 1, + update({'deleted': True, 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupIngressRule).\ filter_by(group_id=security_group_id).\ - update({'deleted': 1, + update({'deleted': True, 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) @@ -1993,11 +1993,11 @@ def security_group_destroy_all(context, session=None): session = get_session() with session.begin(): session.query(models.SecurityGroup).\ - update({'deleted': 1, + update({'deleted': True, 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupIngressRule).\ - update({'deleted': 1, + update({'deleted': True, 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) @@ -2678,7 +2678,7 @@ def instance_metadata_update_or_create(context, instance_id, metadata): meta_ref = models.InstanceMetadata() meta_ref.update({"key": key, "value": value, "instance_id": instance_id, - "deleted": 0}) + "deleted": False}) meta_ref.save(session=session) return metadata -- cgit From 8747611e4bd69b6da204b2c021fd5400c961db1d Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Tue, 7 Jun 2011 10:47:29 -0400 Subject: Removed empty init --- nova/tests/test_libvirt.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py index d0bdaa738..8b4183164 100644 --- a/nova/tests/test_libvirt.py +++ b/nova/tests/test_libvirt.py @@ -210,10 +210,8 @@ class LibvirtConnTestCase(test.TestCase): connection.LibvirtConnection._conn = fake def fake_lookup(self, instance_name): - class FakeVirtDomain(object): - def __init__(self): - pass + class FakeVirtDomain(object): def snapshotCreateXML(self, *args): return None -- cgit From 7bae412d230171baf1ba7bec7262705404d1ed7f Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Tue, 7 Jun 2011 10:47:14 -0500 Subject: Add the option to specify a default IPv6 gateway. --- bin/nova-manage | 13 +++++++++---- nova/network/manager.py | 11 +++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index b0cd343f5..7f024f9ca 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -96,6 +96,7 @@ flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('vlan_start', 'nova.network.manager') flags.DECLARE('vpn_start', 'nova.network.manager') flags.DECLARE('fixed_range_v6', 'nova.network.manager') +flags.DECLARE('gateway_v6', 'nova.network.manager') flags.DECLARE('images_path', 'nova.image.local') flags.DECLARE('libvirt_type', 'nova.virt.libvirt.connection') flags.DEFINE_flag(flags.HelpFlag()) @@ -545,13 +546,14 @@ class FloatingIpCommands(object): class NetworkCommands(object): """Class for managing networks.""" - def create(self, fixed_range=None, num_networks=None, - network_size=None, vlan_start=None, - vpn_start=None, fixed_range_v6=None, label='public'): + def create(self, fixed_range=None, num_networks=None, network_size=None, + vlan_start=None, vpn_start=None, fixed_range_v6=None, + gateway_v6=None, label='public'): """Creates fixed ips for host by range arguments: fixed_range=FLAG, [num_networks=FLAG], [network_size=FLAG], [vlan_start=FLAG], - [vpn_start=FLAG], [fixed_range_v6=FLAG]""" + [vpn_start=FLAG], [fixed_range_v6=FLAG], + [gateway_v6=FLAG]""" if not fixed_range: msg = _('Fixed range in the form of 10.0.0.0/8 is ' 'required to create networks.') @@ -567,6 +569,8 @@ class NetworkCommands(object): vpn_start = FLAGS.vpn_start if not fixed_range_v6: fixed_range_v6 = FLAGS.fixed_range_v6 + if not gateway_v6: + gateway_v6 = FLAGS.gateway_v6 net_manager = utils.import_object(FLAGS.network_manager) try: net_manager.create_networks(context.get_admin_context(), @@ -576,6 +580,7 @@ class NetworkCommands(object): vlan_start=int(vlan_start), vpn_start=int(vpn_start), cidr_v6=fixed_range_v6, + gateway_v6=gateway_v6, label=label) except ValueError, e: print e diff --git a/nova/network/manager.py b/nova/network/manager.py index f726c4b26..b5352ca0f 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -86,6 +86,7 @@ flags.DEFINE_string('floating_range', '4.4.4.0/24', 'Floating IP address block') flags.DEFINE_string('fixed_range', '10.0.0.0/8', 'Fixed IP address block') flags.DEFINE_string('fixed_range_v6', 'fd00::/48', 'Fixed IPv6 address block') +flags.DEFINE_string('gateway_v6', None, 'Default IPv6 gateway') flags.DEFINE_integer('cnt_vpn_clients', 0, 'Number of addresses reserved for vpn clients') flags.DEFINE_string('network_driver', 'nova.network.linux_net', @@ -292,7 +293,7 @@ class NetworkManager(manager.SchedulerDependentManager): return host def create_networks(self, context, cidr, num_networks, network_size, - cidr_v6, label, *args, **kwargs): + cidr_v6, gateway_v6, label, *args, **kwargs): """Create networks based on parameters.""" fixed_net = IPy.IP(cidr) fixed_net_v6 = IPy.IP(cidr_v6) @@ -324,7 +325,13 @@ class NetworkManager(manager.SchedulerDependentManager): significant_bits_v6) net['cidr_v6'] = cidr_v6 project_net_v6 = IPy.IP(cidr_v6) - net['gateway_v6'] = str(project_net_v6[1]) + + if gateway_v6: + # use a pre-defined gateway if one is provided + net['gateway_v6'] = str(gateway_v6) + else: + net['gateway_v6'] = str(project_net_v6[1]) + net['netmask_v6'] = str(project_net_v6.prefixlen()) network_ref = self.db.network_create_safe(context, net) -- cgit From aa343c994c4738374bd91531ae2e260175690a56 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Tue, 7 Jun 2011 11:45:25 -0500 Subject: Remove unnecessary docstrings. --- bin/nova-manage | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 7f024f9ca..0147ae21b 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -549,11 +549,7 @@ class NetworkCommands(object): def create(self, fixed_range=None, num_networks=None, network_size=None, vlan_start=None, vpn_start=None, fixed_range_v6=None, gateway_v6=None, label='public'): - """Creates fixed ips for host by range - arguments: fixed_range=FLAG, [num_networks=FLAG], - [network_size=FLAG], [vlan_start=FLAG], - [vpn_start=FLAG], [fixed_range_v6=FLAG], - [gateway_v6=FLAG]""" + """Creates fixed ips for host by range""" if not fixed_range: msg = _('Fixed range in the form of 10.0.0.0/8 is ' 'required to create networks.') -- cgit From e8d6740fefcac3734021edaf53a40ecb145ccaa3 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Tue, 7 Jun 2011 13:47:40 -0400 Subject: DRY up the image_state logic. Fix an issue where glance style images (which aren't required to have an 'image_state' property) couldn't be used to run instances on the EC2 controller. --- nova/api/ec2/cloud.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index ac73cd595..316298c39 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -136,6 +136,13 @@ class CloudController(object): return services[0]['availability_zone'] return 'unknown zone' + def _get_image_state(self, image): + # NOTE(vish): fallback status if image_state isn't set + state = image.get('status') + if state == 'active': + state = 'available' + return image['properties'].get('image_state', state) + def get_metadata(self, address): ctxt = context.get_admin_context() instance_ref = self.compute_api.get_all(ctxt, fixed_ip=address) @@ -896,14 +903,13 @@ class CloudController(object): ramdisk = self._get_image(context, kwargs['ramdisk_id']) kwargs['ramdisk_id'] = ramdisk['id'] image = self._get_image(context, kwargs['image_id']) - if not image: + + if image: + image_state = self._get_image_state(image) + else: raise exception.ImageNotFound(image_id=kwargs['image_id']) - try: - available = (image['properties']['image_state'] == 'available') - except KeyError: - available = False - if not available: + if image_state != 'available': raise exception.ApiError(_('Image must be available')) instances = self.compute_api.create(context, @@ -1021,11 +1027,8 @@ class CloudController(object): get('image_location'), name) else: i['imageLocation'] = image['properties'].get('image_location') - # NOTE(vish): fallback status if image_state isn't set - state = image.get('status') - if state == 'active': - state = 'available' - i['imageState'] = image['properties'].get('image_state', state) + + i['imageState'] = self._get_image_state(image) i['displayName'] = name i['description'] = image.get('description') display_mapping = {'aki': 'kernel', -- cgit From 641f16a5343ca5d95ea10ec5031a27a7f131c337 Mon Sep 17 00:00:00 2001 From: Alex Meade Date: Tue, 7 Jun 2011 15:17:34 -0400 Subject: pep8 --- nova/api/direct.py | 2 +- nova/utils.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/api/direct.py b/nova/api/direct.py index ea7425e19..ec79151b1 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -324,7 +324,7 @@ class Limited(object): def __init__(self, proxy): self._proxy = proxy - if not self.__doc__: #pylint: disable=E0203 + if not self.__doc__: # pylint: disable=E0203 self.__doc__ = proxy.__doc__ if not self._allowed: self._allowed = [] diff --git a/nova/utils.py b/nova/utils.py index e77c80262..691134ada 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -142,7 +142,7 @@ def execute(*cmd, **kwargs): env = os.environ.copy() if addl_env: env.update(addl_env) - _PIPE = subprocess.PIPE #pylint: disable=E1101 + _PIPE = subprocess.PIPE # pylint: disable=E1101 obj = subprocess.Popen(cmd, stdin=_PIPE, stdout=_PIPE, @@ -153,8 +153,8 @@ def execute(*cmd, **kwargs): result = obj.communicate(process_input) else: result = obj.communicate() - obj.stdin.close() #pylint: disable=E1101 - _returncode = obj.returncode #pylint: disable=E1101 + obj.stdin.close() # pylint: disable=E1101 + _returncode = obj.returncode # pylint: disable=E1101 if _returncode: LOG.debug(_('Result was %s') % _returncode) if type(check_exit_code) == types.IntType \ -- cgit From 8f93aa59aca5440a4d9668942703bf235379ed59 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Tue, 7 Jun 2011 16:05:03 -0400 Subject: Added test_run_instances_image_status_active to test_cloud. --- nova/tests/test_cloud.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index a58e8bc39..ba133c860 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -487,6 +487,21 @@ class CloudTestCase(test.TestCase): self.assertRaises(exception.ApiError, run_instances, self.context, **kwargs) + def test_run_instances_image_status_active(self): + kwargs = {'image_id': FLAGS.default_image, + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1} + run_instances = self.cloud.run_instances + + def fake_show_stat_active(self, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, + 'type': 'machine'}, 'status': 'active'} + + self.stubs.Set(local.LocalImageService, 'show', fake_show_stat_active) + + result = run_instances(self.context, **kwargs) + self.assertEqual(len(result['instancesSet']), 1) + def test_terminate_instances(self): inst1 = db.instance_create(self.context, {'reservation_id': 'a', 'image_ref': 1, -- cgit From d4742cf8505ff86a4732f8d198fe6cedf260898e Mon Sep 17 00:00:00 2001 From: Alex Meade Date: Tue, 7 Jun 2011 16:08:25 -0400 Subject: Added virtual environment to PEP8 tests --- run_tests.sh | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/run_tests.sh b/run_tests.sh index 9aa555484..c7bcd5d67 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -67,14 +67,11 @@ function run_pep8 { srcfiles=`find bin -type f ! -name "nova.conf*"` srcfiles+=" `find tools/*`" srcfiles+=" nova setup.py plugins/xenserver/xenapi/etc/xapi.d/plugins/glance" - pep8 --repeat --show-pep8 --show-source --exclude=vcsversion.py ${srcfiles} + # Just run PEP8 in current environment + ${wrapper} pep8 --repeat --show-pep8 --show-source \ + --exclude=vcsversion.py ${srcfiles} } -if [ $just_pep8 -eq 1 ]; then - run_pep8 - exit -fi - NOSETESTS="python run_tests.py $noseargs" if [ $never_venv -eq 0 ] @@ -103,6 +100,11 @@ then fi fi +if [ $just_pep8 -eq 1 ]; then + run_pep8 + exit +fi + run_tests || exit # Also run pep8 if no options were provided. -- cgit From a90974347dd396990d8e6fadeac15abd07cb19ea Mon Sep 17 00:00:00 2001 From: John Tran Date: Tue, 7 Jun 2011 14:36:40 -0700 Subject: adding Authorizer key for ImportPublicKey --- nova/api/ec2/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 1915d007d..890d57fe7 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -242,6 +242,7 @@ class Authorizer(wsgi.Middleware): 'CreateKeyPair': ['all'], 'DeleteKeyPair': ['all'], 'DescribeSecurityGroups': ['all'], + 'ImportPublicKey': ['all'], 'AuthorizeSecurityGroupIngress': ['netadmin'], 'RevokeSecurityGroupIngress': ['netadmin'], 'CreateSecurityGroup': ['netadmin'], -- cgit