From 7c8096384507908a5e583f4554d0fc765ae5f2eb Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Thu, 27 Jan 2011 20:39:33 +0900 Subject: adding testcode --- nova/compute/manager.py | 66 ++-- nova/db/sqlalchemy/api.py | 49 +-- nova/db/sqlalchemy/models.py | 12 +- nova/scheduler/driver.py | 53 ++-- nova/scheduler/manager.py | 28 +- nova/tests/test_compute.py | 305 ++++++++++++++++++ nova/tests/test_scheduler.py | 722 +++++++++++++++++++++++++++++++++++++++++++ nova/tests/test_service.py | 61 +++- nova/tests/test_virt.py | 520 ++++++++++++++++++++++++++++++- nova/virt/fake.py | 12 +- nova/virt/libvirt_conn.py | 159 +++++----- nova/virt/xenapi_conn.py | 14 +- nova/volume/manager.py | 2 +- 13 files changed, 1807 insertions(+), 196 deletions(-) (limited to 'nova') diff --git a/nova/compute/manager.py b/nova/compute/manager.py index efb5753aa..4acba7153 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -125,12 +125,12 @@ class ComputeManager(manager.Manager): """Insert compute node specific information to DB.""" try: - service_ref = db.service_get_by_args(ctxt, - host, - binary) + service_ref = self.db.service_get_by_args(ctxt, + host, + binary) except exception.NotFound: - msg = _(("""Cannot insert compute manager specific info""" - """Because no service record found.""")) + msg = _(("""Cannot insert compute manager specific info,""" + """ Because no service record found.""")) raise exception.Invalid(msg) # Updating host information @@ -141,14 +141,14 @@ class ComputeManager(manager.Manager): version = self.driver.get_hypervisor_version() cpu_info = self.driver.get_cpu_info() - db.service_update(ctxt, - service_ref['id'], - {'vcpus': vcpu, - 'memory_mb': memory_mb, - 'local_gb': local_gb, - 'hypervisor_type': hypervisor, - 'hypervisor_version': version, - 'cpu_info': cpu_info}) + self.db.service_update(ctxt, + service_ref['id'], + {'vcpus': vcpu, + 'memory_mb': memory_mb, + 'local_gb': local_gb, + 'hypervisor_type': hypervisor, + 'hypervisor_version': version, + 'cpu_info': cpu_info}) def _update_state(self, context, instance_id): """Update the state of an instance from the driver info.""" @@ -596,22 +596,22 @@ class ComputeManager(manager.Manager): """ Check the host cpu is compatible to a cpu given by xml.""" return self.driver.compare_cpu(cpu_info) - def pre_live_migration(self, context, instance_id, dest): + def pre_live_migration(self, context, instance_id): """Any preparation for live migration at dst host.""" # Getting instance info - instance_ref = db.instance_get(context, instance_id) + instance_ref = self.db.instance_get(context, instance_id) ec2_id = instance_ref['hostname'] # Getting fixed ips - fixed_ip = db.instance_get_fixed_address(context, instance_id) + fixed_ip = self.db.instance_get_fixed_address(context, instance_id) if not fixed_ip: msg = _('%s(%s) doesnt have fixed_ip') % (instance_id, ec2_id) raise exception.NotFound(msg) # If any volume is mounted, prepare here. if len(instance_ref['volumes']) == 0: - logging.info(_("%s has no volume.") % ec2_id) + LOG.info(_("%s has no volume."), ec2_id) else: for v in instance_ref['volumes']: self.volume_manager.setup_compute_volume(context, v['id']) @@ -634,7 +634,7 @@ class ComputeManager(manager.Manager): """executes live migration.""" # Get instance for error handling. - instance_ref = db.instance_get(context, instance_id) + instance_ref = self.db.instance_get(context, instance_id) ec2_id = instance_ref['hostname'] try: @@ -647,27 +647,27 @@ class ComputeManager(manager.Manager): "args": {'instance_id': instance_id}}) # Asking dest host to preparing live migration. - compute_topic = db.queue_get_for(context, - FLAGS.compute_topic, - dest) + compute_topic = self.db.queue_get_for(context, + FLAGS.compute_topic, + dest) rpc.call(context, - compute_topic, - {"method": "pre_live_migration", - "args": {'instance_id': instance_id, - 'dest': dest}}) + compute_topic, + {"method": "pre_live_migration", + "args": {'instance_id': instance_id}}) except Exception, e: + print e msg = _('Pre live migration for %s failed at %s') - logging.error(msg, ec2_id, dest) - db.instance_set_state(context, - instance_id, - power_state.RUNNING, - 'running') + LOG.error(msg, ec2_id, dest) + self.db.instance_set_state(context, + instance_id, + power_state.RUNNING, + 'running') for v in instance_ref['volumes']: - db.volume_update(context, - v['id'], - {'status': 'in-use'}) + self.db.volume_update(context, + v['id'], + {'status': 'in-use'}) # e should be raised. just calling "raise" may raise NotFound. raise e diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 248a46f65..1cdd5a286 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -897,41 +897,42 @@ def instance_get_all_by_host(context, hostname): @require_context -def _instance_get_sum_by_host_and_project(context, column, hostname, proj_id): +def instance_get_vcpu_sum_by_host_and_project(context, hostname, proj_id): session = get_session() - result = session.query(models.Instance).\ - filter_by(host=hostname).\ - filter_by(project_id=proj_id).\ - filter_by(deleted=can_read_deleted(context)).\ - value(column) - if not result: + filter_by(host=hostname).\ + filter_by(project_id=proj_id).\ + filter_by(deleted=False).\ + value(func.sum(models.Instance.vcpus)) + if None == result: return 0 return result -@require_context -def instance_get_vcpu_sum_by_host_and_project(context, hostname, proj_id): - return _instance_get_sum_by_host_and_project(context, - 'vcpus', - hostname, - proj_id) - - @require_context def instance_get_memory_sum_by_host_and_project(context, hostname, proj_id): - return _instance_get_sum_by_host_and_project(context, - 'memory_mb', - hostname, - proj_id) - + session = get_session() + result = session.query(models.Instance).\ + filter_by(host=hostname).\ + filter_by(project_id=proj_id).\ + filter_by(deleted=False).\ + value(func.sum(models.Instance.memory_mb)) + if None == result: + return 0 + return result @require_context def instance_get_disk_sum_by_host_and_project(context, hostname, proj_id): - return _instance_get_sum_by_host_and_project(context, - 'local_gb', - hostname, - proj_id) + session = get_session() + result = session.query(models.Instance).\ + filter_by(host=hostname).\ + filter_by(project_id=proj_id).\ + filter_by(deleted=False).\ + value(func.sum(models.Instance.local_gb)) + if None == result: + return 0 + return result + @require_context diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index b28c64b59..7c40d5596 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -161,11 +161,11 @@ class Service(BASE, NovaBase): # The below items are compute node only. # -1 or None is inserted for other service. - vcpus = Column(Integer, nullable=False, default=-1) - memory_mb = Column(Integer, nullable=False, default=-1) - local_gb = Column(Integer, nullable=False, default=-1) - hypervisor_type = Column(String(128)) - hypervisor_version = Column(Integer, nullable=False, default=-1) + vcpus = Column(Integer, nullable=True) + memory_mb = Column(Integer, nullable=True) + local_gb = Column(Integer, nullable=True) + hypervisor_type = Column(String(128), nullable=True) + hypervisor_version = Column(Integer, nullable=True) # Note(masumotok): Expected Strings example: # # '{"arch":"x86_64", "model":"Nehalem", @@ -174,7 +174,7 @@ class Service(BASE, NovaBase): # # Points are "json translatable" and it must have all # dictionary keys above. - cpu_info = Column(String(512)) + cpu_info = Column(Text(), nullable=True) class Certificate(BASE, NovaBase): diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 65745093b..d4ad42388 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -69,11 +69,10 @@ class Scheduler(object): raise NotImplementedError(_("Must implement a fallback schedule")) def schedule_live_migration(self, context, instance_id, dest): - """ live migration method """ + """live migration method""" # Whether instance exists and running instance_ref = db.instance_get(context, instance_id) - ec2_id = instance_ref['hostname'] # Checking instance. self._live_migration_src_check(context, instance_ref) @@ -159,48 +158,45 @@ class Scheduler(object): def _live_migration_common_check(self, context, instance_ref, dest): """ - Live migration check routine. - Below pre-checkings are followed by - http://wiki.libvirt.org/page/TodoPreMigrationChecks + Live migration check routine. + Below pre-checkings are followed by + http://wiki.libvirt.org/page/TodoPreMigrationChecks """ # Checking dest exists. dservice_refs = db.service_get_all_by_host(context, dest) if len(dservice_refs) <= 0: - msg = _('%s does not exists.') - raise exception.Invalid(msg % dest) + raise exception.Invalid(_('%s does not exists.') % dest) dservice_ref = dservice_refs[0] # Checking original host( where instance was launched at) exists. - orighost = instance_ref['launched_on'] - oservice_refs = db.service_get_all_by_host(context, orighost) + oservice_refs = db.service_get_all_by_host(context, + instance_ref['launched_on']) if len(oservice_refs) <= 0: msg = _('%s(where instance was launched at) does not exists.') - raise exception.Invalid(msg % orighost) + raise exception.Invalid(msg % instance_ref['launched_on']) oservice_ref = oservice_refs[0] # Checking hypervisor is same. - otype = oservice_ref['hypervisor_type'] - dtype = dservice_ref['hypervisor_type'] - if otype != dtype: + if oservice_ref['hypervisor_type'] != dservice_ref['hypervisor_type']: msg = _('Different hypervisor type(%s->%s)') - raise exception.Invalid(msg % (otype, dtype)) + raise exception.Invalid(msg % (oservice_ref['hypervisor_type'], + dservice_ref['hypervisor_type'])) # Checkng hypervisor version. - oversion = oservice_ref['hypervisor_version'] - dversion = dservice_ref['hypervisor_version'] - if oversion > dversion: + if oservice_ref['hypervisor_version'] > \ + dservice_ref['hypervisor_version']: msg = _('Older hypervisor version(%s->%s)') - raise exception.Invalid(msg % (oversion, dversion)) + raise exception.Invalid(msg % (oservice_ref['hypervisor_version'], + dservice_ref['hypervisor_version'])) # Checking cpuinfo. - cpu_info = oservice_ref['cpu_info'] try: rpc.call(context, db.queue_get_for(context, FLAGS.compute_topic, dest), {"method": 'compare_cpu', - "args": {'cpu_info': cpu_info}}) + "args": {'cpu_info': oservice_ref['cpu_info']}}) except rpc.RemoteError, e: msg = _(("""%s doesnt have compatibility to %s""" @@ -211,7 +207,7 @@ class Scheduler(object): raise e def has_enough_resource(self, context, instance_ref, dest): - """ Check if destination host has enough resource for live migration""" + """Check if destination host has enough resource for live migration""" # Getting instance information ec2_id = instance_ref['hostname'] @@ -222,28 +218,27 @@ class Scheduler(object): # Gettin host information service_refs = db.service_get_all_by_host(context, dest) if len(service_refs) <= 0: - msg = _('%s does not exists.') - raise exception.Invalid(msg % dest) + raise exception.Invalid(_('%s does not exists.') % dest) service_ref = service_refs[0] total_cpu = int(service_ref['vcpus']) total_mem = int(service_ref['memory_mb']) total_hdd = int(service_ref['local_gb']) - instances_ref = db.instance_get_all_by_host(context, dest) - for i_ref in instances_ref: + instances_refs = db.instance_get_all_by_host(context, dest) + for i_ref in instances_refs: total_cpu -= int(i_ref['vcpus']) total_mem -= int(i_ref['memory_mb']) total_hdd -= int(i_ref['local_gb']) # Checking host has enough information - logging.debug('host(%s) remains vcpu:%s mem:%s hdd:%s,' % + logging.debug(_('host(%s) remains vcpu:%s mem:%s hdd:%s,') % (dest, total_cpu, total_mem, total_hdd)) - logging.debug('instance(%s) has vcpu:%s mem:%s hdd:%s,' % + logging.debug(_('instance(%s) has vcpu:%s mem:%s hdd:%s,') % (ec2_id, vcpus, mem, hdd)) if total_cpu <= vcpus or total_mem <= mem or total_hdd <= hdd: - msg = '%s doesnt have enough resource for %s' % (dest, ec2_id) - raise exception.NotEmpty(msg) + raise exception.NotEmpty(_('%s is not capable to migrate %s') % + (dest, ec2_id)) logging.debug(_('%s has_enough_resource() for %s') % (dest, ec2_id)) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 1cc767a03..a181225a6 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -73,17 +73,13 @@ class SchedulerManager(manager.Manager): # Based on bear design summit discussion, # just put this here for bexar release. def show_host_resource(self, context, host, *args): - """ show the physical/usage resource given by hosts.""" + """show the physical/usage resource given by hosts.""" - services = db.service_get_all_by_host(context, host) - if len(services) == 0: - return {'ret': False, 'msg': 'No such Host'} - - compute = [s for s in services if s['topic'] == 'compute'] - if 0 == len(compute): - service_ref = services[0] - else: - service_ref = compute[0] + computes = db.service_get_all_compute_sorted(context) + computes = [s for s,v in computes if s['host'] == host] + if 0 == len(computes): + return {'ret': False, 'msg': 'No such Host or not compute node.'} + service_ref = computes[0] # Getting physical resource information h_resource = {'vcpus': service_ref['vcpus'], @@ -92,13 +88,15 @@ class SchedulerManager(manager.Manager): # Getting usage resource information u_resource = {} - instances_ref = db.instance_get_all_by_host(context, - service_ref['host']) + instances_refs = db.instance_get_all_by_host(context, + service_ref['host']) - if 0 == len(instances_ref): - return {'ret': True, 'phy_resource': h_resource, 'usage': {}} + if 0 == len(instances_refs): + return {'ret': True, + 'phy_resource': h_resource, + 'usage': u_resource} - project_ids = [i['project_id'] for i in instances_ref] + project_ids = [i['project_id'] for i in instances_refs] project_ids = list(set(project_ids)) for p_id in project_ids: vcpus = db.instance_get_vcpu_sum_by_host_and_project(context, diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 09f6ee94a..344c2d2b5 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -20,6 +20,7 @@ Tests For Compute """ import datetime +import mox from nova import compute from nova import context @@ -27,9 +28,12 @@ from nova import db from nova import exception from nova import flags from nova import log as logging +from nova import rpc from nova import test from nova import utils from nova.auth import manager +from nova.compute import manager as compute_manager +from nova.compute import power_state LOG = logging.getLogger('nova.tests.compute') @@ -219,3 +223,304 @@ class ComputeTestCase(test.TestCase): self.assertEqual(ret_val, None) self.compute.terminate_instance(self.context, instance_id) + + def test_update_service_exception(self): + """nova-compute updates Serivce table on DB like below. + nova.service.Serivce.start -> + nova.compute.ComputeManager.update_service. + This testcase confirms if no record found on Service + table, exception can be raised. + """ + host = 'foo' + binary = 'nova-compute' + dbmock = self.mox.CreateMock(db) + dbmock.service_get_by_args(mox.IgnoreArg(), + mox.StrContains(host), + mox.StrContains(binary)).\ + AndRaise(exception.NotFound()) + self.compute.db = dbmock + self.mox.ReplayAll() + try: + self.compute.update_service('dummy', host, binary) + except exception.Invalid, e: + msg = 'Cannot insert compute manager specific info' + c1 = ( 0 <= e.message.find(msg)) + self.assertTrue(c1) + self.mox.ResetAll() + + def test_update_service_success(self): + """nova-compute updates Serivce table on DB like below. + nova.service.Serivce.start -> + nova.compute.ComputeManager.update_service. + In this method, vcpus/memory_mb/local_gb/hypervisor_type/ + hypervisor_version/cpu_info should be changed. + Based on this specification, this testcase confirms + if this method finishes successfully, + meaning self.db.service_update is called with dictinary + + {'vcpu':aaa, 'memory_mb':bbb, 'local_gb':ccc, + 'hypervisor_type':ddd, 'hypervisor_version':eee, + 'cpu_info':fff} + + Since each value of above dict can be obtained through + driver(different depends on environment), + only dictionary keys are checked. + """ + + def dic_key_check(dic): + validkey = ['vcpus', 'memory_mb', 'local_gb', + 'hypervisor_type', 'hypervisor_version', 'cpu_info'] + return (list(set(validkey)) == list(set(dic.keys()))) + + host = 'foo' + binary = 'nova-compute' + service_ref = {'id':1, 'binary':'nova-compute', 'topic':'compute'} + dbmock = self.mox.CreateMock(db) + dbmock.service_get_by_args(mox.IgnoreArg(), + mox.StrContains(host), + mox.StrContains(binary)).\ + AndReturn(service_ref) + dbmock.service_update(mox.IgnoreArg(), + service_ref['id'], + mox.Func(dic_key_check)) + + self.compute.db = dbmock + self.mox.ReplayAll() + try: + self.compute.update_service('dummy', host, binary) + except exception.Invalid, e: + msg = 'Cannot insert compute manager specific info' + c1 = ( 0 <= e.message.find(msg)) + self.assertTrue(c1) + self.mox.ResetAll() + + def _setup_other_managers(self): + self.volume_manager = utils.import_object(FLAGS.volume_manager) + self.network_manager = utils.import_object(FLAGS.network_manager) + self.compute_driver = utils.import_object(FLAGS.compute_driver) + + def test_pre_live_migration_instance_has_no_fixed_ip(self): + """ + if instances that are intended to be migrated doesnt have fixed_ip + (not happens usually), pre_live_migration has to raise Exception. + """ + instance_ref={'id':1, 'volumes':[{'id':1}, {'id':2}], + 'hostname':'i-000000001'} + c = context.get_admin_context() + i_id = instance_ref['id'] + + dbmock = self.mox.CreateMock(db) + dbmock.instance_get(c, i_id).AndReturn(instance_ref) + dbmock.instance_get_fixed_address(c, i_id).AndReturn(None) + + self.compute.db = dbmock + self.mox.ReplayAll() + self.assertRaises(exception.NotFound, + self.compute.pre_live_migration, + c, instance_ref['id']) + self.mox.ResetAll() + + def test_pre_live_migration_instance_has_volume(self): + """if any volumes are attached to the instances that are + intended to be migrated, setup_compute_volume must be + called because aoe module should be inserted at destination + host. This testcase checks on it. + """ + instance_ref={'id':1, 'volumes':[{'id':1}, {'id':2}], + 'hostname':'i-000000001'} + c = context.get_admin_context() + i_id=instance_ref['id'] + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + volmock = self.mox.CreateMock(self.volume_manager) + netmock = self.mox.CreateMock(self.network_manager) + drivermock = self.mox.CreateMock(self.compute_driver) + + dbmock.instance_get(c, i_id).AndReturn(instance_ref) + dbmock.instance_get_fixed_address(c, i_id).AndReturn('dummy') + for i in range(len(instance_ref['volumes'])): + vid = instance_ref['volumes'][i]['id'] + volmock.setup_compute_volume(c, vid).InAnyOrder('g1') + netmock.setup_compute_network(c, instance_ref['id']) + drivermock.ensure_filtering_rules_for_instance(instance_ref) + + self.compute.db = dbmock + self.compute.volume_manager = volmock + self.compute.network_manager = netmock + self.compute.driver = drivermock + + self.mox.ReplayAll() + ret = self.compute.pre_live_migration(c, i_id) + self.assertEqual(ret, None) + self.mox.ResetAll() + + def test_pre_live_migration_instance_has_no_volume(self): + """if any volumes are not attached to the instances that are + intended to be migrated, log message should be appears + because administrator can proove instance conditions before + live_migration if any trouble occurs. + """ + instance_ref={'id':1, 'volumes':[], 'hostname':'i-20000001'} + c = context.get_admin_context() + i_id = instance_ref['id'] + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + netmock = self.mox.CreateMock(self.network_manager) + drivermock = self.mox.CreateMock(self.compute_driver) + + dbmock.instance_get(c, i_id).AndReturn(instance_ref) + dbmock.instance_get_fixed_address(c, i_id).AndReturn('dummy') + self.mox.StubOutWithMock(compute_manager.LOG, 'info') + compute_manager.LOG.info(_("%s has no volume."), instance_ref['hostname']) + netmock.setup_compute_network(c, i_id) + drivermock.ensure_filtering_rules_for_instance(instance_ref) + + self.compute.db = dbmock + self.compute.network_manager = netmock + self.compute.driver = drivermock + + self.mox.ReplayAll() + ret = self.compute.pre_live_migration(c, i_id) + self.assertEqual(ret, None) + self.mox.ResetAll() + + def test_live_migration_instance_has_volume(self): + """Any volumes are mounted by instances to be migrated are found, + vblade health must be checked before starting live-migration. + And that is checked by check_for_export(). + This testcase confirms check_for_export() is called. + """ + instance_ref={'id':1, 'volumes':[{'id':1}, {'id':2}], 'hostname':'i-00000001'} + c = context.get_admin_context() + dest='dummydest' + i_id = instance_ref['id'] + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + drivermock = self.mox.CreateMock(self.compute_driver) + + dbmock.instance_get(c, instance_ref['id']).AndReturn(instance_ref) + self.mox.StubOutWithMock(rpc, 'call') + rpc.call(c, FLAGS.volume_topic, + {"method": "check_for_export", + "args": {'instance_id': i_id}}).InAnyOrder('g1') + rpc.call(c, db.queue_get_for(c, FLAGS.compute_topic, dest), + {"method": "pre_live_migration", + "args": {'instance_id': i_id}}).InAnyOrder('g1') + + self.compute.db = dbmock + self.compute.driver = drivermock + self.mox.ReplayAll() + ret = self.compute.live_migration(c, i_id, dest) + self.assertEqual(ret, None) + self.mox.ResetAll() + + def test_live_migration_instance_has_volume_and_exception(self): + """In addition to test_live_migration_instance_has_volume testcase, + this testcase confirms if any exception raises from check_for_export(). + Then, valid seaquence of this method should recovering instance/volumes + status(ex. instance['state_description'] is changed from 'migrating' + -> 'running', was changed by scheduler) + """ + instance_ref={'id':1, 'volumes':[{'id':1}, {'id':2}], + 'hostname':'i-000000001'} + dest='dummydest' + c = context.get_admin_context() + i_id = instance_ref['id'] + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + drivermock = self.mox.CreateMock(self.compute_driver) + + dbmock.instance_get(c, instance_ref['id']).AndReturn(instance_ref) + self.mox.StubOutWithMock(rpc, 'call') + rpc.call(c, FLAGS.volume_topic, + {"method": "check_for_export", + "args": {'instance_id': i_id}}).InAnyOrder('g1') + compute_topic = db.queue_get_for(c, FLAGS.compute_topic, dest) + dbmock.queue_get_for(c, FLAGS.compute_topic, dest).AndReturn(compute_topic) + rpc.call(c, db.queue_get_for(c, FLAGS.compute_topic, dest), + {"method": "pre_live_migration", + "args": {'instance_id': i_id}}).\ + InAnyOrder('g1').AndRaise(rpc.RemoteError('du', 'mm', 'y')) + self.mox.StubOutWithMock(compute_manager.LOG, 'error') + compute_manager.LOG.error('Pre live migration for %s failed at %s', + instance_ref['hostname'], dest) + dbmock.instance_set_state(c, i_id, power_state.RUNNING, 'running') + for i in range(len(instance_ref['volumes'])): + vid = instance_ref['volumes'][i]['id'] + dbmock.volume_update(c, vid, {'status': 'in-use'}) + + self.compute.db = dbmock + self.compute.driver = drivermock + self.mox.ReplayAll() + self.assertRaises(rpc.RemoteError, + self.compute.live_migration, + c, i_id, dest) + self.mox.ResetAll() + + def test_live_migration_instance_has_no_volume_and_exception(self): + """Simpler than test_live_migration_instance_has_volume_and_exception""" + + instance_ref={'id':1, 'volumes':[], 'hostname':'i-000000001'} + dest='dummydest' + c = context.get_admin_context() + i_id = instance_ref['id'] + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + drivermock = self.mox.CreateMock(self.compute_driver) + + dbmock.instance_get(c, instance_ref['id']).AndReturn(instance_ref) + self.mox.StubOutWithMock(rpc, 'call') + compute_topic = db.queue_get_for(c, FLAGS.compute_topic, dest) + dbmock.queue_get_for(c, FLAGS.compute_topic, dest).AndReturn(compute_topic) + rpc.call(c, compute_topic, + {"method": "pre_live_migration", + "args": {'instance_id': i_id}}).\ + AndRaise(rpc.RemoteError('du', 'mm', 'y')) + self.mox.StubOutWithMock(compute_manager.LOG, 'error') + compute_manager.LOG.error('Pre live migration for %s failed at %s', + instance_ref['hostname'], dest) + dbmock.instance_set_state(c, i_id, power_state.RUNNING, 'running') + + self.compute.db = dbmock + self.compute.driver = drivermock + self.mox.ReplayAll() + self.assertRaises(rpc.RemoteError, + self.compute.live_migration, + c, i_id, dest) + self.mox.ResetAll() + + def test_live_migration_instance_has_volume(self): + """Simpler version than test_live_migration_instance_has_volume.""" + instance_ref={'id':1, 'volumes':[{'id':1}, {'id':2}], + 'hostname':'i-000000001'} + c = context.get_admin_context() + dest='dummydest' + i_id = instance_ref['id'] + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + drivermock = self.mox.CreateMock(self.compute_driver) + + dbmock.instance_get(c, i_id).AndReturn(instance_ref) + self.mox.StubOutWithMock(rpc, 'call') + rpc.call(c, FLAGS.volume_topic, + {"method": "check_for_export", + "args": {'instance_id': i_id}}).InAnyOrder('g1') + compute_topic = db.queue_get_for(c, FLAGS.compute_topic, dest) + dbmock.queue_get_for(c, FLAGS.compute_topic, dest).AndReturn(compute_topic) + rpc.call(c, compute_topic, + {"method": "pre_live_migration", + "args": {'instance_id': i_id}}).InAnyOrder('g1') + drivermock.live_migration(c, instance_ref, dest) + + self.compute.db = dbmock + self.compute.driver = drivermock + self.mox.ReplayAll() + ret = self.compute.live_migration(c, i_id, dest) + self.assertEqual(ret, None) + self.mox.ResetAll() diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 9d458244b..c62bca9b1 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -20,10 +20,12 @@ Tests For Scheduler """ import datetime +import mox from mox import IgnoreArg from nova import context from nova import db +from nova import exception from nova import flags from nova import service from nova import test @@ -32,6 +34,8 @@ from nova import utils from nova.auth import manager as auth_manager from nova.scheduler import manager from nova.scheduler import driver +from nova.compute import power_state +from nova.db.sqlalchemy import models FLAGS = flags.FLAGS @@ -75,7 +79,102 @@ class SchedulerTestCase(test.TestCase): 'args': {'num': 7}}) self.mox.ReplayAll() scheduler.named_method(ctxt, 'topic', num=7) + + def test_show_host_resource_host_not_exit(self): + """ + A testcase of driver.has_enough_resource + given host does not exists. + """ + scheduler = manager.SchedulerManager() + dest = 'dummydest' + ctxt = context.get_admin_context() + + self.mox.StubOutWithMock(manager, 'db', use_mock_anything=True) + manager.db.service_get_all_compute_sorted(mox.IgnoreArg()).\ + AndReturn([]) + + self.mox.ReplayAll() + result = scheduler.show_host_resource(ctxt, dest) + # ret should be dict + keys = ['ret', 'msg'] + c1 = list(set(result.keys())) == list(set(keys)) + c2 = not result['ret'] + c3 = result['msg'].find('No such Host or not compute node') <= 0 + self.assertTrue( c1 and c2 and c3) + self.mox.UnsetStubs() + + def test_show_host_resource_no_project(self): + """ + A testcase of driver.show_host_resource + no instance stays on the given host + """ + scheduler = manager.SchedulerManager() + dest = 'dummydest' + ctxt = context.get_admin_context() + r0 = {'vcpus':16, 'memory_mb':32, 'local_gb':100} + service_ref = {'id':1, 'host':dest} + service_ref.update(r0) + + self.mox.StubOutWithMock(manager, 'db', use_mock_anything=True) + manager.db.service_get_all_compute_sorted(mox.IgnoreArg()).\ + AndReturn([(service_ref, 0)]) + manager.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([]) + + self.mox.ReplayAll() + result = scheduler.show_host_resource(ctxt, dest) + # ret should be dict + keys = ['ret', 'phy_resource', 'usage'] + c1 = list(set(result.keys())) == list(set(keys)) + c2 = result['ret'] + c3 = result['phy_resource'] == r0 + c4 = result['usage'] == {} + self.assertTrue( c1 and c2 and c3 and c4) + self.mox.UnsetStubs() + + def test_show_host_resource_works_correctly(self): + """ + A testcase of driver.show_host_resource + to make sure everything finished with no error. + """ + scheduler = manager.SchedulerManager() + dest = 'dummydest' + ctxt = context.get_admin_context() + r0 = {'vcpus':16, 'memory_mb':32, 'local_gb':100} + r1 = {'vcpus':10, 'memory_mb':4, 'local_gb':20} + r2 = {'vcpus':10, 'memory_mb':20, 'local_gb':30} + service_ref = {'id':1, 'host':dest} + service_ref.update(r0) + instance_ref2 = {'id':2, 'project_id':'p-01', 'host':'dummy'} + instance_ref2.update(r1) + instance_ref3 = {'id':3, 'project_id':'p-02', 'host':'dummy'} + instance_ref3.update(r1) + self.mox.StubOutWithMock(manager, 'db', use_mock_anything=True) + manager.db.service_get_all_compute_sorted(mox.IgnoreArg()).\ + AndReturn([(service_ref, 0)]) + manager.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([instance_ref2, instance_ref3]) + for p in ['p-01', 'p-02']: + manager.db.instance_get_vcpu_sum_by_host_and_project( + ctxt, dest, p).AndReturn(r2['vcpus']) + manager.db.instance_get_memory_sum_by_host_and_project( + ctxt, dest, p).AndReturn(r2['memory_mb']) + manager.db.instance_get_disk_sum_by_host_and_project( + ctxt, dest, p).AndReturn(r2['local_gb']) + + self.mox.ReplayAll() + result = scheduler.show_host_resource(ctxt, dest) + # ret should be dict + keys = ['ret', 'phy_resource', 'usage'] + c1 = list(set(result.keys())) == list(set(keys)) + c2 = result['ret'] + c3 = result['phy_resource'] == r0 + c4 = result['usage'].keys() == ['p-01', 'p-02'] + c5 = result['usage']['p-01'] == r2 + c6 = result['usage']['p-02'] == r2 + self.assertTrue( c1 and c2 and c3 and c4 and c5 and c6) + self.mox.UnsetStubs() class ZoneSchedulerTestCase(test.TestCase): """Test case for zone scheduler""" @@ -384,3 +483,626 @@ class SimpleDriverTestCase(test.TestCase): volume2.delete_volume(self.context, volume_id) volume1.kill() volume2.kill() + + def test_scheduler_live_migraiton_with_volume(self): + """ + driver.scheduler_live_migration finishes successfully + (volumes are attached to instances) + This testcase make sure schedule_live_migration + changes instance state from 'running' -> 'migrating' + """ + driver_i = self.scheduler.driver + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-00000001', 'host':'dummy', + 'volumes':[{'id':1}, {'id':2}]} + dest = 'dummydest' + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + # must be IgnoreArg() because scheduler changes ctxt's memory address + driver.db.instance_get(mox.IgnoreArg(), i_ref['id']).AndReturn(i_ref) + + self.mox.StubOutWithMock(driver_i, '_live_migration_src_check') + driver_i._live_migration_src_check(mox.IgnoreArg(), i_ref) + self.mox.StubOutWithMock(driver_i, '_live_migration_dest_check') + driver_i._live_migration_dest_check(mox.IgnoreArg(), i_ref, dest) + self.mox.StubOutWithMock(driver_i, '_live_migration_common_check') + driver_i._live_migration_common_check(mox.IgnoreArg(), i_ref, dest) + driver.db.instance_set_state(mox.IgnoreArg(), i_ref['id'], + power_state.PAUSED, 'migrating') + for v in i_ref['volumes']: + driver.db.volume_update(mox.IgnoreArg(), v['id'], + {'status': 'migrating'}) + self.mox.StubOutWithMock(rpc, 'cast', use_mock_anything=True) + kwargs={'instance_id':i_ref['id'], 'dest':dest} + rpc.cast(ctxt, db.queue_get_for(ctxt, topic, i_ref['host']), + {"method": 'live_migration', + "args": kwargs}) + + self.mox.ReplayAll() + self.scheduler.live_migration(ctxt, topic, + instance_id=i_ref['id'], dest=dest) + self.mox.UnsetStubs() + + def test_scheduler_live_migraiton_no_volume(self): + """ + driver.scheduler_live_migration finishes successfully + (volumes are attached to instances) + This testcase make sure schedule_live_migration + changes instance state from 'running' -> 'migrating' + """ + driver_i = self.scheduler.driver + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', 'volumes':[]} + dest = 'dummydest' + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + # must be IgnoreArg() because scheduler changes ctxt's memory address + driver.db.instance_get(mox.IgnoreArg(), i_ref['id']).AndReturn(i_ref) + self.mox.StubOutWithMock(driver_i, '_live_migration_src_check') + driver_i._live_migration_src_check(mox.IgnoreArg(), i_ref) + self.mox.StubOutWithMock(driver_i, '_live_migration_dest_check') + driver_i._live_migration_dest_check(mox.IgnoreArg(), i_ref, dest) + self.mox.StubOutWithMock(driver_i, '_live_migration_common_check') + driver_i._live_migration_common_check(mox.IgnoreArg(), i_ref, dest) + driver.db.instance_set_state(mox.IgnoreArg(), i_ref['id'], + power_state.PAUSED, 'migrating') + self.mox.StubOutWithMock(rpc, 'cast', use_mock_anything=True) + kwargs={'instance_id':i_ref['id'], 'dest':dest} + rpc.cast(ctxt, db.queue_get_for(ctxt, topic, i_ref['host']), + {"method": 'live_migration', + "args": kwargs}) + + self.mox.ReplayAll() + self.scheduler.live_migration(ctxt, topic, + instance_id=i_ref['id'], dest=dest) + self.mox.UnsetStubs() + + def test_live_migraiton_src_check_instance_not_running(self): + """ + A testcase of driver._live_migration_src_check. + The instance given by instance_id is not running. + """ + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + dest = 'dummydest' + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', + 'volumes':[], 'state_description':'migrating', + 'state':power_state.RUNNING} + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_src_check(ctxt, i_ref) + except exception.Invalid, e: + self.assertTrue(e.message.find('is not running') > 0) + self.mox.UnsetStubs() + + def test_live_migraiton_src_check_volume_node_not_alive(self): + """ + A testcase of driver._live_migration_src_check. + Volume node is not alive if any volumes are attached to + the given instance. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', + 'volumes':[{'id':1}, {'id':2}], + 'state_description':'running', 'state':power_state.RUNNING} + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_topic(mox.IgnoreArg(), 'volume').\ + AndReturn([]) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_src_check(ctxt, i_ref) + except exception.Invalid, e: + self.assertTrue(e.message.find('volume node is not alive') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_src_check_volume_node_not_alive(self): + """ + A testcase of driver._live_migration_src_check. + The testcase make sure src-compute node is alive. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', 'volumes':[], + 'state_description':'running', 'state':power_state.RUNNING} + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_topic(mox.IgnoreArg(), 'compute').\ + AndReturn([]) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_src_check(ctxt, i_ref) + except exception.Invalid, e: + self.assertTrue(e.message.find('is not alive') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_src_check_works_correctly(self): + """ + A testcase of driver._live_migration_src_check. + The testcase make sure everything finished with no error. + """ + driver_i = self.scheduler.driver + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', 'volumes':[], + 'state_description':'running', 'state':power_state.RUNNING} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('host', i_ref['host']) + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_topic(mox.IgnoreArg(), 'compute').\ + AndReturn([service_ref]) + self.mox.StubOutWithMock(driver_i, 'service_is_up') + driver_i.service_is_up(service_ref).AndReturn(True) + + self.mox.ReplayAll() + ret = driver_i._live_migration_src_check(ctxt, i_ref) + self.assertTrue(ret == None) + self.mox.UnsetStubs() + + def test_live_migraiton_dest_check_service_not_exists(self): + """ + A testcase of driver._live_migration_dst_check. + Destination host does not exist. + """ + driver_i = self.scheduler.driver + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('host', i_ref['host']) + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([]) + + self.mox.ReplayAll() + try: + driver_i._live_migration_dest_check(ctxt, i_ref, dest) + except exception.Invalid, e: + self.assertTrue(e.message.find('does not exists') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_dest_check_service_isnot_compute(self): + """ + A testcase of driver._live_migration_dst_check. + Destination host does not provide compute. + """ + driver_i = self.scheduler.driver + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('host', i_ref['host']) + service_ref.__setitem__('topic', 'api') + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + + self.mox.ReplayAll() + try: + driver_i._live_migration_dest_check(ctxt, i_ref, dest) + except exception.Invalid, e: + self.assertTrue(e.message.find('must be compute node') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_dest_check_service_not_alive(self): + """ + A testcase of driver._live_migration_dst_check. + Destination host compute service is not alive. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('host', i_ref['host']) + service_ref.__setitem__('topic', 'compute') + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + self.mox.StubOutWithMock(self.scheduler.driver, 'service_is_up') + self.scheduler.driver.service_is_up(service_ref).AndReturn(False) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_dest_check(ctxt, i_ref, dest) + except exception.Invalid, e: + self.assertTrue(e.message.find('is not alive') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_dest_check_service_same_host(self): + """ + A testcase of driver._live_migration_dst_check. + Destination host is same as src host. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummydest'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('host', i_ref['host']) + service_ref.__setitem__('topic', 'compute') + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + self.mox.StubOutWithMock(self.scheduler.driver, 'service_is_up') + self.scheduler.driver.service_is_up(service_ref).AndReturn(True) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_dest_check(ctxt, i_ref, dest) + except exception.Invalid, e: + self.assertTrue(e.message.find('is running now. choose other host') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_dest_check_service_works_correctly(self): + """ + A testcase of driver._live_migration_dst_check. + The testcase make sure everything finished with no error. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummydest'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('host', i_ref['host']) + service_ref.__setitem__('topic', 'compute') + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + self.mox.StubOutWithMock(self.scheduler.driver, 'service_is_up') + self.scheduler.driver.service_is_up(service_ref).AndReturn(True) + self.mox.StubOutWithMock(self.scheduler.driver, 'has_enough_resource') + self.scheduler.driver.has_enough_resource(mox.IgnoreArg(), i_ref, dest) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_dest_check(ctxt, i_ref, dest) + except exception.Invalid, e: + self.assertTrue(e.message.find('is running now. choose other host') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_common_check_service_dest_not_exists(self): + """ + A testcase of driver._live_migration_common_check. + Destination host does not exist. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([]) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_common_check(ctxt, + i_ref, + dest) + except exception.Invalid, e: + self.assertTrue(e.message.find('does not exists') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_common_check_service_orig_not_exists(self): + """ + A testcase of driver._live_migration_common_check. + Original host(an instance launched on) does not exist. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', 'launched_on':'h1'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('topic', 'compute') + service_ref.__setitem__('host', i_ref['host']) + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.service_get_all_by_host(mox.IgnoreArg(), + i_ref['launched_on']).\ + AndReturn([]) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_common_check(ctxt, + i_ref, + dest) + except exception.Invalid, e: + msg = 'where instance was launched at) does not exists' + self.assertTrue(e.message.find(msg) >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_common_check_service_different_hypervisor(self): + """ + A testcase of driver._live_migration_common_check. + Original host and dest host has different hypervisor type. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', + 'host':'dummy', 'launched_on':'h1'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('topic', 'compute') + service_ref.__setitem__('hypervisor_type', 'kvm') + service_ref2 = models.Service() + service_ref2.__setitem__('id', 2) + service_ref2.__setitem__('hypervisor_type', 'xen') + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.service_get_all_by_host(mox.IgnoreArg(), + i_ref['launched_on']).\ + AndReturn([service_ref2]) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_common_check(ctxt, + i_ref, + dest) + except exception.Invalid, e: + msg = 'Different hypervisor type' + self.assertTrue(e.message.find(msg) >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_common_check_service_different_version(self): + """ + A testcase of driver._live_migration_common_check. + Original host and dest host has different hypervisor version. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', + 'host':'dummy', 'launched_on':'h1'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('topic', 'compute') + service_ref.__setitem__('hypervisor_version', 12000) + service_ref2 = models.Service() + service_ref2.__setitem__('id', 2) + service_ref2.__setitem__('hypervisor_version', 12001) + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.service_get_all_by_host(mox.IgnoreArg(), + i_ref['launched_on']).\ + AndReturn([service_ref2]) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_common_check(ctxt, + i_ref, + dest) + except exception.Invalid, e: + msg = 'Older hypervisor version' + self.assertTrue(e.message.find(msg) >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_common_check_service_checking_cpuinfo_fail(self): + """ + A testcase of driver._live_migration_common_check. + Original host and dest host has different hypervisor version. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', + 'host':'dummy', 'launched_on':'h1'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('topic', 'compute') + service_ref.__setitem__('hypervisor_version', 12000) + service_ref2 = models.Service() + service_ref2.__setitem__('id', 2) + service_ref2.__setitem__('hypervisor_version', 12000) + service_ref2.__setitem__('cpuinfo', 'info') + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.service_get_all_by_host(mox.IgnoreArg(), + i_ref['launched_on']).\ + AndReturn([service_ref2]) + driver.db.queue_get_for(mox.IgnoreArg(), FLAGS.compute_topic, dest) + self.mox.StubOutWithMock(driver, 'rpc', use_mock_anything=True) + driver.rpc.call(mox.IgnoreArg(), mox.IgnoreArg(), + {"method": 'compare_cpu', + "args": {'cpu_info': service_ref2['cpu_info']}}).\ + AndRaise(rpc.RemoteError('doesnt have compatibility to', '', '')) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_common_check(ctxt, + i_ref, + dest) + except rpc.RemoteError, e: + msg = 'doesnt have compatibility to' + self.assertTrue(e.message.find(msg) >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_common_check_service_works_correctly(self): + """ + A testcase of driver._live_migration_common_check. + The testcase make sure everything finished with no error. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', + 'host':'dummy', 'launched_on':'h1'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('topic', 'compute') + service_ref.__setitem__('hypervisor_version', 12000) + service_ref2 = models.Service() + service_ref2.__setitem__('id', 2) + service_ref2.__setitem__('hypervisor_version', 12000) + service_ref2.__setitem__('cpuinfo', 'info') + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.service_get_all_by_host(mox.IgnoreArg(), + i_ref['launched_on']).\ + AndReturn([service_ref2]) + driver.db.queue_get_for(mox.IgnoreArg(), FLAGS.compute_topic, dest) + self.mox.StubOutWithMock(driver, 'rpc', use_mock_anything=True) + driver.rpc.call(mox.IgnoreArg(), mox.IgnoreArg(), + {"method": 'compare_cpu', + "args": {'cpu_info': service_ref2['cpu_info']}}) + + self.mox.ReplayAll() + ret = self.scheduler.driver._live_migration_common_check(ctxt, + i_ref, + dest) + self.assertTrue(ret == None) + self.mox.UnsetStubs() + + def test_has_enough_resource_lack_resource_vcpu(self): + """ + A testcase of driver.has_enough_resource. + Lack of vcpu.(boundary check) + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} + i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', + 'vcpus':6, 'memory_mb':8, 'local_gb':10} + i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([i_ref2, i_ref3]) + + self.mox.ReplayAll() + try: + self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) + except exception.NotEmpty, e: + msg = 'is not capable to migrate' + self.assertTrue(e.message.find(msg) >= 0) + self.mox.UnsetStubs() + + def test_has_enough_resource_lack_resource_memory(self): + """ + A testcase of driver.has_enough_resource. + Lack of memory_mb.(boundary check) + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} + i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':16, 'local_gb':10} + i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([i_ref2, i_ref3]) + + self.mox.ReplayAll() + try: + self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) + except exception.NotEmpty, e: + msg = 'is not capable to migrate' + self.assertTrue(e.message.find(msg) >= 0) + self.mox.UnsetStubs() + self.mox.UnsetStubs() + + def test_has_enough_resource_lack_resource_disk(self): + """ + A testcase of driver.has_enough_resource. + Lack of local_gb.(boundary check) + """ + scheduler = manager.SchedulerManager() + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} + i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':80} + i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([i_ref2, i_ref3]) + + self.mox.ReplayAll() + try: + self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) + except exception.NotEmpty, e: + msg = 'is not capable to migrate' + self.assertTrue(e.message.find(msg) >= 0) + self.mox.UnsetStubs() + + def test_has_enough_resource_works_correctly(self): + """ + A testcase of driver.has_enough_resource + to make sure everything finished with no error. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} + i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([i_ref2, i_ref3]) + + self.mox.ReplayAll() + ret = self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) + self.assertTrue(ret == None) + self.mox.UnsetStubs() diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index a67c8d1e8..a147e69b4 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -30,6 +30,7 @@ from nova import rpc from nova import test from nova import service from nova import manager +from nova.compute import manager as compute_manager FLAGS = flags.FLAGS flags.DEFINE_string("fake_manager", "nova.tests.test_service.FakeManager", @@ -41,7 +42,20 @@ class FakeManager(manager.Manager): def test_method(self): return 'manager' - +# temporary variable to store host/binary/self.mox from each method to fake class. +global_host = None +global_binary = None +global_mox = None +class FakeComputeManager(compute_manager.ComputeManager): + """Fake computemanager manager for tests""" + + def __init__(self, compute_driver=None, *args, **kwargs): + global ghost, gbinary, gmox + self.update_service(mox.IgnoreArg(), mox.StrContains(ghost), mox.StrContains(gbinary)) + gmox.ReplayAll() + super(FakeComputeManager, self).__init__(compute_driver, *args, **kwargs) + + class ExtendedService(service.Service): def test_method(self): return 'service' @@ -258,3 +272,48 @@ class ServiceTestCase(test.TestCase): serv.report_state() self.assert_(not serv.model_disconnected) + + def test_compute_can_update_services(self): + """ + Test nova-compute successfully updated Service table on DB. + Doing so, self.manager.update_service must be called + if 'self.binary == nova-compute', and this testcase checks on it. + """ + host = 'foo' + binary = 'nova-compute' + topic = 'compute1' + service_create = {'host': host, + 'binary': binary, + 'topic': topic, + 'report_count': 0, + 'availability_zone': 'nova'} + service_ref = {'host': host, + 'binary': binary, + 'topic': topic, + 'report_count': 0, + 'availability_zone': 'nova', + 'id': 1} + + service.db.service_get_by_args(mox.IgnoreArg(), + host, + binary).AndRaise(exception.NotFound()) + service.db.service_create(mox.IgnoreArg(), + service_create).AndReturn(service_ref) + self.mox.StubOutWithMock(compute_manager.ComputeManager, 'update_service') + + + global ghost, gbinary, gmox + ghost = host + gbinary = binary + gmox = self.mox + + serv = service.Service(host, + binary, + topic, + 'nova.tests.test_service.FakeComputeManager') + # ReplayAll has been executed FakeComputeManager.__init__() + #self.mox.ReplayAll() + serv.start() + serv.stop() + + diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index afdc89ba2..177e8f021 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -14,21 +14,29 @@ # License for the specific language governing permissions and limitations # under the License. +import mox + from xml.etree.ElementTree import fromstring as xml_to_tree from xml.dom.minidom import parseString as xml_to_dom from nova import context from nova import db +from nova import exception from nova import flags from nova import test +from nova import logging from nova import utils from nova.api.ec2 import cloud from nova.auth import manager +from nova.db.sqlalchemy import models +from nova.compute import power_state from nova.virt import libvirt_conn FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') +libvirt = None +libxml2 = None class LibvirtConnTestCase(test.TestCase): def setUp(self): @@ -52,6 +60,38 @@ class LibvirtConnTestCase(test.TestCase): 'bridge': 'br101', 'instance_type': 'm1.small'} + def _driver_dependent_test_setup(self): + """ + Setup method. + Call this method at the top of each testcase method, + if the testcase is necessary libvirt and cheetah. + """ + try : + global libvirt + global libxml2 + libvirt_conn.libvirt = __import__('libvirt') + libvirt_conn.libxml2 = __import__('libxml2') + libvirt_conn._late_load_cheetah() + libvirt = __import__('libvirt') + except ImportError, e: + logging.warn("""This test has not been done since """ + """using driver-dependent library Cheetah/libvirt/libxml2.""") + raise e + + # inebitable mocks for calling + #nova.virt.libvirt_conn.LibvirtConnection.__init__ + nwmock = self.mox.CreateMock(libvirt_conn.NWFilterFirewall) + self.mox.StubOutWithMock(libvirt_conn, 'NWFilterFirewall', + use_mock_anything=True) + libvirt_conn.NWFilterFirewall(mox.IgnoreArg()).AndReturn(nwmock) + + obj = utils.import_object(FLAGS.firewall_driver) + fwmock = self.mox.CreateMock(obj) + self.mox.StubOutWithMock(libvirt_conn, 'utils', + use_mock_anything=True) + libvirt_conn.utils.import_object(FLAGS.firewall_driver).AndReturn(fwmock) + return nwmock, fwmock + def test_xml_and_uri_no_ramdisk_no_kernel(self): instance_data = dict(self.test_instance) self._check_xml_and_uri(instance_data, @@ -188,9 +228,8 @@ class LibvirtConnTestCase(test.TestCase): expected_result, '%s failed common check %d' % (xml, i)) - # This test is supposed to make sure we don't override a specifically - # set uri - # + # This test is supposed to make sure we don't override a specifically set uri + # # Deliberately not just assigning this string to FLAGS.libvirt_uri and # checking against that later on. This way we make sure the # implementation doesn't fiddle around with the FLAGS. @@ -202,6 +241,480 @@ class LibvirtConnTestCase(test.TestCase): uri = conn.get_uri() self.assertEquals(uri, testuri) + def test_get_memory_mb(self): + """ + Check if get_memory_mb returns memory value + Connection/OS/driver differenct does not matter for this method, + so everyone can execute for checking. + """ + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertTrue(0 < conn.get_memory_mb()) + self.mox.UnsetStubs() + + def test_get_cpu_info_works_correctly(self): + """ + Check if get_cpu_info works correctly. + (in case libvirt.getCapabilities() works correctly) + """ + xml=("""x86_64Nehalem""" + """Intel""" + """""" + """""" + """""" + """""" + """""" + """""" + """""") + + try: + self._driver_dependent_test_setup() + except: + return + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', use_mock_anything=True) + libvirt_conn.LibvirtConnection._conn.getCapabilities().AndReturn(xml) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertTrue(0 < len(conn.get_cpu_info())) + self.mox.UnsetStubs() + + def test_get_cpu_info_inappropreate_xml(self): + """ + Check if get_cpu_info raises exception + in case libvirt.getCapabilities() returns wrong xml + (in case of xml doesnt have tag) + """ + xml=("""x86_64Nehalem""" + """Intel""" + """""" + """""" + """""" + """""" + """""" + """""" + """""") + + try: + self._driver_dependent_test_setup() + except: + return + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', use_mock_anything=True) + libvirt_conn.LibvirtConnection._conn.getCapabilities().AndReturn(xml) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + try: + conn.get_cpu_info() + except exception.Invalid, e: + c1 = ( 0 <= e.message.find('Invalid xml') ) + self.assertTrue(c1) + self.mox.UnsetStubs() + + def test_get_cpu_info_inappropreate_xml2(self): + """ + Check if get_cpu_info raises exception + in case libvirt.getCapabilities() returns wrong xml + (in case of xml doesnt have inproper tag + meaning missing "socket" attribute) + """ + xml=("""x86_64Nehalem""" + """Intel""" + """""" + """""" + """""" + """""" + """""" + """""" + """""") + + try: + self._driver_dependent_test_setup() + except: + return + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', use_mock_anything=True) + libvirt_conn.LibvirtConnection._conn.getCapabilities().AndReturn(xml) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + try: + conn.get_cpu_info() + except exception.Invalid, e: + c1 = ( 0 <= e.message.find('Invalid xml: topology') ) + self.assertTrue(c1) + self.mox.UnsetStubs() + + def test_compare_cpu_works_correctly(self): + """Calling libvirt.compute_cpu() and works correctly """ + + t = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ + """"topology":{"cores":"%s", "threads":"%s", """ + """"sockets":"%s"}, "features":[%s]}""") + cpu_info = t % ('x86', 'model', 'vendor', '2', '1', '4', '"tm"') + + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', use_mock_anything=True) + libvirt_conn.LibvirtConnection._conn.compareCPU(mox.IgnoreArg(),0).AndReturn(1) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertTrue( None== conn.compare_cpu(cpu_info)) + self.mox.UnsetStubs() + + def test_compare_cpu_raises_exception(self): + """ + Libvirt-related exception occurs when calling + libvirt.compare_cpu(). + """ + t = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ + """"topology":{"cores":"%s", "threads":"%s", """ + """"sockets":"%s"}, "features":[%s]}""") + cpu_info = t % ('x86', 'model', 'vendor', '2', '1', '4', '"tm"') + + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', + use_mock_anything=True) + libvirt_conn.LibvirtConnection._conn.compareCPU(mox.IgnoreArg(),0).\ + AndRaise(libvirt.libvirtError('ERR')) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertRaises(libvirt.libvirtError, conn.compare_cpu, cpu_info) + self.mox.UnsetStubs() + + def test_compare_cpu_no_compatibility(self): + """libvirt.compare_cpu() return less than 0.(no compatibility)""" + + t = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ + """"topology":{"cores":"%s", "threads":"%s", """ + """"sockets":"%s"}, "features":[%s]}""") + cpu_info = t % ('x86', 'model', 'vendor', '2', '1', '4', '"tm"') + + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', + use_mock_anything=True) + libvirt_conn.LibvirtConnection._conn.compareCPU(mox.IgnoreArg(),0).\ + AndRaise(exception.Invalid('ERR')) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertRaises(exception.Invalid, conn.compare_cpu, cpu_info) + self.mox.UnsetStubs() + + def test_ensure_filtering_rules_for_instance_works_correctly(self): + """ensure_filtering_rules_for_instance works as expected correctly""" + + instance_ref = models.Instance() + instance_ref.__setitem__('id', 1) + + try: + nwmock, fwmock = self._driver_dependent_test_setup() + except: + return + + nwmock.setup_basic_filtering(mox.IgnoreArg()) + fwmock.prepare_instance_filter(instance_ref) + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', + use_mock_anything=True) + n = 'nova-instance-%s' % instance_ref.name + libvirt_conn.LibvirtConnection._conn.nwfilterLookupByName(n) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + conn.ensure_filtering_rules_for_instance(instance_ref) + self.mox.UnsetStubs() + + def test_ensure_filtering_rules_for_instance_timeout(self): + """ensure_filtering_fules_for_instance finishes with timeout""" + + instance_ref = models.Instance() + instance_ref.__setitem__('id', 1) + + try: + nwmock, fwmock = self._driver_dependent_test_setup() + except: + return + + nwmock.setup_basic_filtering(mox.IgnoreArg()) + fwmock.prepare_instance_filter(instance_ref) + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', + use_mock_anything=True) + n = 'nova-instance-%s' % instance_ref.name + for i in range(FLAGS.live_migration_timeout_sec * 2): + libvirt_conn.LibvirtConnection._conn.\ + nwfilterLookupByName(n).AndRaise(libvirt.libvirtError('ERR')) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + try: + conn.ensure_filtering_rules_for_instance(instance_ref) + except exception.Error, e: + c1 = ( 0<=e.message.find('Timeout migrating for')) + self.assertTrue(c1) + self.mox.UnsetStubs() + + def test_live_migration_works_correctly(self): + """_live_migration works as expected correctly """ + + class dummyCall(object): + f = None + def start(self, interval=0, now=False): + pass + + instance_ref = models.Instance() + instance_ref.__setitem__('id', 1) + dest = 'desthost' + ctxt = context.get_admin_context() + + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', + use_mock_anything=True) + vdmock = self.mox.CreateMock(libvirt.virDomain) + self.mox.StubOutWithMock(vdmock, "migrateToURI", + use_mock_anything=True) + vdmock.migrateToURI(FLAGS.live_migration_uri % dest, mox.IgnoreArg(), + None, FLAGS.live_migration_bandwidth).\ + AndReturn(None) + libvirt_conn.LibvirtConnection._conn.lookupByName(instance_ref.name).\ + AndReturn(vdmock) + # below description is also ok. + #self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection._conn, + # "lookupByName", use_mock_anything=True) + + libvirt_conn.utils.LoopingCall(f=None).AndReturn(dummyCall()) + + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + ret = conn._live_migration(ctxt, instance_ref, dest) + self.assertTrue(ret == None) + self.mox.UnsetStubs() + + def test_live_migration_raises_exception(self): + """ + _live_migration raises exception, then this testcase confirms + state_description/state for the instances/volumes are recovered. + """ + class Instance(models.NovaBase): + id = 0 + volumes = None + name = 'name' + + ctxt = context.get_admin_context() + dest = 'desthost' + instance_ref = Instance() + instance_ref.__setitem__('id', 1) + instance_ref.__setitem__('volumes', [{'id':1}, {'id':2}]) + + try: + nwmock, fwmock = self._driver_dependent_test_setup() + except: + return + + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', + use_mock_anything=True) + vdmock = self.mox.CreateMock(libvirt.virDomain) + self.mox.StubOutWithMock(vdmock, "migrateToURI", + use_mock_anything=True) + vdmock.migrateToURI(FLAGS.live_migration_uri % dest, mox.IgnoreArg(), + None, FLAGS.live_migration_bandwidth).\ + AndRaise(libvirt.libvirtError('ERR')) + libvirt_conn.LibvirtConnection._conn.lookupByName(instance_ref.name).\ + AndReturn(vdmock) + self.mox.StubOutWithMock(db, 'instance_set_state') + db.instance_set_state(ctxt, instance_ref['id'], + power_state.RUNNING, 'running') + self.mox.StubOutWithMock(db, 'volume_update') + for v in instance_ref.volumes: + db.volume_update(ctxt, v['id'], {'status': 'in-use'}).\ + InAnyOrder('g1') + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertRaises(libvirt.libvirtError, + conn._live_migration, + ctxt, instance_ref, dest) + self.mox.UnsetStubs() + + def test_post_live_migration_working_correctly(self): + """_post_live_migration works as expected correctly """ + + dest = 'dummydest' + ctxt = context.get_admin_context() + instance_ref = {'id':1, 'hostname':'i-00000001', 'host':dest, + 'fixed_ip':'dummyip', 'floating_ip':'dummyflip', + 'volumes':[{'id':1}, {'id':2} ]} + network_ref = {'id':1, 'host':dest} + floating_ip_ref = {'id':1, 'address':'1.1.1.1'} + + try: + nwmock, fwmock = self._driver_dependent_test_setup() + except: + return + fwmock.unfilter_instance(instance_ref) + + fixed_ip = instance_ref['fixed_ip'] + self.mox.StubOutWithMock(db, 'instance_get_fixed_address') + db.instance_get_fixed_address(ctxt, instance_ref['id']).AndReturn(fixed_ip) + self.mox.StubOutWithMock(db, 'fixed_ip_update') + db.fixed_ip_update(ctxt, fixed_ip, {'host': dest}) + self.mox.StubOutWithMock(db, 'fixed_ip_get_network') + db.fixed_ip_get_network(ctxt, fixed_ip).AndReturn(network_ref) + self.mox.StubOutWithMock(db, 'network_update') + db.network_update(ctxt, network_ref['id'], {'host': dest}) + + fl_ip = instance_ref['floating_ip'] + self.mox.StubOutWithMock(db, 'instance_get_floating_address') + db.instance_get_floating_address(ctxt, instance_ref['id']).AndReturn(fl_ip) + self.mox.StubOutWithMock(db, 'floating_ip_get_by_address') + db.floating_ip_get_by_address(ctxt, instance_ref['floating_ip']).\ + AndReturn(floating_ip_ref) + self.mox.StubOutWithMock(db, 'floating_ip_update') + db.floating_ip_update(ctxt, floating_ip_ref['address'], {'host': dest}) + + self.mox.StubOutWithMock(db, 'instance_update') + db.instance_update(ctxt, instance_ref['id'], + {'state_description': 'running', + 'state': power_state.RUNNING, 'host': dest}) + self.mox.StubOutWithMock(db, 'volume_update') + for v in instance_ref['volumes']: + db.volume_update(ctxt, v['id'], {'status': 'in-use'}) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + conn._post_live_migration( ctxt, instance_ref, dest) + self.mox.UnsetStubs() + + def test_post_live_migration_no_floating_ip(self): + """ + _post_live_migration works as expected correctly + (in case instance doesnt have floaitng ip) + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + instance_ref = {'id':1, 'hostname':'i-00000001', 'host':dest, + 'fixed_ip':'dummyip', 'floating_ip':'dummyflip', + 'volumes':[{'id':1}, {'id':2} ]} + network_ref = {'id':1, 'host':dest} + floating_ip_ref = {'id':1, 'address':'1.1.1.1'} + + try: + nwmock, fwmock = self._driver_dependent_test_setup() + except: + return + fwmock.unfilter_instance(instance_ref) + + fixed_ip = instance_ref['fixed_ip'] + self.mox.StubOutWithMock(db, 'instance_get_fixed_address') + db.instance_get_fixed_address(ctxt, instance_ref['id']).AndReturn(fixed_ip) + self.mox.StubOutWithMock(db, 'fixed_ip_update') + db.fixed_ip_update(ctxt, fixed_ip, {'host': dest}) + self.mox.StubOutWithMock(db, 'fixed_ip_get_network') + db.fixed_ip_get_network(ctxt, fixed_ip).AndReturn(network_ref) + self.mox.StubOutWithMock(db, 'network_update') + db.network_update(ctxt, network_ref['id'], {'host': dest}) + + self.mox.StubOutWithMock(db, 'instance_get_floating_address') + db.instance_get_floating_address(ctxt, instance_ref['id']).AndReturn(None) + self.mox.StubOutWithMock(libvirt_conn.LOG, 'info') + libvirt_conn.LOG.info(_('post livemigration operation is started..')) + libvirt_conn.LOG.info(_('floating_ip is not found for %s'), + instance_ref['hostname']) + # Checking last messages are ignored. may be no need to check so strictly? + libvirt_conn.LOG.info(mox.IgnoreArg()) + libvirt_conn.LOG.info(mox.IgnoreArg()) + + self.mox.StubOutWithMock(db, 'instance_update') + db.instance_update(ctxt, instance_ref['id'], + {'state_description': 'running', + 'state': power_state.RUNNING, + 'host': dest}) + self.mox.StubOutWithMock(db, 'volume_update') + for v in instance_ref['volumes']: + db.volume_update(ctxt, v['id'], {'status': 'in-use'}) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + conn._post_live_migration( ctxt, instance_ref, dest) + self.mox.UnsetStubs() + + def test_post_live_migration_no_floating_ip_with_exception(self): + """ + _post_live_migration works as expected correctly + (in case instance doesnt have floaitng ip, and raise exception) + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + instance_ref = {'id':1, 'hostname':'i-00000001', 'host':dest, + 'fixed_ip':'dummyip', 'floating_ip':'dummyflip', + 'volumes':[{'id':1}, {'id':2} ]} + network_ref = {'id':1, 'host':dest} + floating_ip_ref = {'id':1, 'address':'1.1.1.1'} + + try: + nwmock, fwmock = self._driver_dependent_test_setup() + except: + return + fwmock.unfilter_instance(instance_ref) + + fixed_ip = instance_ref['fixed_ip'] + self.mox.StubOutWithMock(db, 'instance_get_fixed_address') + db.instance_get_fixed_address(ctxt, instance_ref['id']).AndReturn(fixed_ip) + self.mox.StubOutWithMock(db, 'fixed_ip_update') + db.fixed_ip_update(ctxt, fixed_ip, {'host': dest}) + self.mox.StubOutWithMock(db, 'fixed_ip_get_network') + db.fixed_ip_get_network(ctxt, fixed_ip).AndReturn(network_ref) + self.mox.StubOutWithMock(db, 'network_update') + db.network_update(ctxt, network_ref['id'], {'host': dest}) + + self.mox.StubOutWithMock(db, 'instance_get_floating_address') + db.instance_get_floating_address(ctxt, instance_ref['id']).\ + AndRaise(exception.NotFound()) + self.mox.StubOutWithMock(libvirt_conn.LOG, 'info') + libvirt_conn.LOG.info(_('post livemigration operation is started..')) + libvirt_conn.LOG.info(_('floating_ip is not found for %s'), + instance_ref['hostname']) + # the last message is ignored. may be no need to check so strictly? + libvirt_conn.LOG.info(mox.IgnoreArg()) + libvirt_conn.LOG.info(mox.IgnoreArg()) + + self.mox.StubOutWithMock(db, 'instance_update') + db.instance_update(ctxt, instance_ref['id'], + {'state_description': 'running', + 'state': power_state.RUNNING, 'host': dest}) + self.mox.StubOutWithMock(db, 'volume_update') + for v in instance_ref['volumes']: + db.volume_update(ctxt, v['id'], {'status': 'in-use'}) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + conn._post_live_migration( ctxt, instance_ref, dest) + self.mox.UnsetStubs() + def tearDown(self): super(LibvirtConnTestCase, self).tearDown() self.manager.delete_project(self.project) @@ -475,3 +988,4 @@ class NWFilterTestCase(test.TestCase): self.fw.prepare_instance_filter(instance) _ensure_all_called() self.teardown_security_group() + diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 80ae7f34c..f469af681 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -316,15 +316,15 @@ class FakeConnection(object): def get_vcpu_number(self): """This method is supported only libvirt. """ - return -1 + return def get_memory_mb(self): """This method is supported only libvirt..""" - return -1 + return def get_local_gb(self): """This method is supported only libvirt..""" - return -1 + return def get_hypervisor_type(self): """This method is supported only libvirt..""" @@ -332,12 +332,16 @@ class FakeConnection(object): def get_hypervisor_version(self): """This method is supported only libvirt..""" - return -1 + return def compare_cpu(self, xml): """This method is supported only libvirt..""" raise NotImplementedError('This method is supported only libvirt.') + def ensure_filtering_rules_for_instance(self, instance_ref): + """This method is supported only libvirt..""" + raise NotImplementedError('This method is supported only libvirt.') + def live_migration(self, context, instance_ref, dest): """This method is supported only libvirt..""" raise NotImplementedError('This method is supported only libvirt.') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 7d1f76b32..49dd03c57 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -861,18 +861,18 @@ class LibvirtConnection(object): def get_cpu_info(self): """ Get cpuinfo information """ - xmlstr = self._conn.getCapabilities() - xml = libxml2.parseDoc(xmlstr) + xml = self._conn.getCapabilities() + xml = libxml2.parseDoc(xml) nodes = xml.xpathEval('//cpu') if len(nodes) != 1: - msg = 'Unexpected xml format. tag "cpu" must be 1, but %d.' \ - % len(nodes) + msg = 'Invalid xml. "" must be 1, but %d.' % len(nodes) msg += '\n' + xml.serialize() raise exception.Invalid(_(msg)) - arch = xml.xpathEval('//cpu/arch')[0].getContent() - model = xml.xpathEval('//cpu/model')[0].getContent() - vendor = xml.xpathEval('//cpu/vendor')[0].getContent() + cpu_info = dict() + cpu_info['arch'] = xml.xpathEval('//cpu/arch')[0].getContent() + cpu_info['model'] = xml.xpathEval('//cpu/model')[0].getContent() + cpu_info['vendor'] = xml.xpathEval('//cpu/vendor')[0].getContent() topology_node = xml.xpathEval('//cpu/topology')[0].get_properties() topology = dict() @@ -890,18 +890,19 @@ class LibvirtConnection(object): feature_nodes = xml.xpathEval('//cpu/feature') features = list() for nodes in feature_nodes: - feature_name = nodes.get_properties().getContent() - features.append(feature_name) + features.append(nodes.get_properties().getContent()) template = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ """"topology":{"cores":"%s", "threads":"%s", """ """"sockets":"%s"}, "features":[%s]}""") - c = topology['cores'] - s = topology['sockets'] - t = topology['threads'] f = ['"%s"' % x for x in features] - cpu_info = template % (arch, model, vendor, c, s, t, ', '.join(f)) - return cpu_info + return template % (cpu_info['arch'], + cpu_info['model'], + cpu_info['vendor'], + topology['cores'], + topology['sockets'], + topology['threads'], + ', '.join(f)) def block_stats(self, instance_name, disk): """ @@ -935,12 +936,12 @@ class LibvirtConnection(object): def compare_cpu(self, cpu_info): """ - Check the host cpu is compatible to a cpu given by xml. - "xml" must be a part of libvirt.openReadonly().getCapabilities(). - return values follows by virCPUCompareResult. - if 0 > return value, do live migration. + Check the host cpu is compatible to a cpu given by xml. + "xml" must be a part of libvirt.openReadonly().getCapabilities(). + return values follows by virCPUCompareResult. + if 0 > return value, do live migration. - 'http://libvirt.org/html/libvirt-libvirt.html#virCPUCompareResult' + 'http://libvirt.org/html/libvirt-libvirt.html#virCPUCompareResult' """ msg = _('Checking cpu_info: instance was launched this cpu.\n: %s ') LOG.info(msg % cpu_info) @@ -952,7 +953,7 @@ class LibvirtConnection(object): url = 'http://libvirt.org/html/libvirt-libvirt.html' url += '#virCPUCompareResult\n' msg = 'CPU does not have compativility.\n' - msg += 'result:%d \n' + msg += 'result:%s \n' msg += 'Refer to %s' msg = _(msg) @@ -960,7 +961,7 @@ class LibvirtConnection(object): try: ret = self._conn.compareCPU(xml, 0) except libvirt.libvirtError, e: - LOG.error(msg % (ret, url)) + LOG.error(msg % (e.message, url)) raise e if ret <= 0: @@ -969,24 +970,26 @@ class LibvirtConnection(object): return def ensure_filtering_rules_for_instance(self, instance_ref): - """ Setting up inevitable filtering rules on compute node, - and waiting for its completion. - To migrate an instance, filtering rules to hypervisors - and firewalls are inevitable on destination host. - ( Waiting only for filterling rules to hypervisor, - since filtering rules to firewall rules can be set faster). - - Concretely, the below method must be called. - - setup_basic_filtering (for nova-basic, etc.) - - prepare_instance_filter(for nova-instance-instance-xxx, etc.) - - to_xml may have to be called since it defines PROJNET, PROJMASK. - but libvirt migrates those value through migrateToURI(), - so , no need to be called. - - Don't use thread for this method since migration should - not be started when setting-up filtering rules operations - are not completed.""" + """ + Setting up inevitable filtering rules on compute node, + and waiting for its completion. + To migrate an instance, filtering rules to hypervisors + and firewalls are inevitable on destination host. + ( Waiting only for filterling rules to hypervisor, + since filtering rules to firewall rules can be set faster). + + Concretely, the below method must be called. + - setup_basic_filtering (for nova-basic, etc.) + - prepare_instance_filter(for nova-instance-instance-xxx, etc.) + + to_xml may have to be called since it defines PROJNET, PROJMASK. + but libvirt migrates those value through migrateToURI(), + so , no need to be called. + + Don't use thread for this method since migration should + not be started when setting-up filtering rules operations + are not completed. + """ # Tf any instances never launch at destination host, # basic-filtering must be set here. @@ -1009,40 +1012,44 @@ class LibvirtConnection(object): raise exception.Error(msg % (ec2_id, instance_ref.name)) time.sleep(0.5) - def live_migration(self, context, instance_ref, dest): + def live_migration(self, ctxt, instance_ref, dest): """ - Just spawning live_migration operation for - distributing high-load. + Just spawning live_migration operation for + distributing high-load. """ - greenthread.spawn(self._live_migration, context, instance_ref, dest) + greenthread.spawn(self._live_migration, ctxt, instance_ref, dest) - def _live_migration(self, context, instance_ref, dest): + def _live_migration(self, ctxt, instance_ref, dest): """ Do live migration.""" # Do live migration. try: - duri = FLAGS.live_migration_uri % dest - flaglist = FLAGS.live_migration_flag.split(',') flagvals = [getattr(libvirt, x.strip()) for x in flaglist] logical_sum = reduce(lambda x, y: x | y, flagvals) - bandwidth = FLAGS.live_migration_bandwidth - if self.read_only: tmpconn = self._connect(self.libvirt_uri, False) dom = tmpconn.lookupByName(instance_ref.name) - dom.migrateToURI(duri, logical_sum, None, bandwidth) + dom.migrateToURI(FLAGS.live_migration_uri % dest, + logical_sum, + None, + FLAGS.live_migration_bandwidth) tmpconn.close() else: dom = self._conn.lookupByName(instance_ref.name) - dom.migrateToURI(duri, logical_sum, None, bandwidth) + dom.migrateToURI(FLAGS.live_migration_uri % dest, + logical_sum, + None, + FLAGS.live_migration_bandwidth) except Exception, e: - id = instance_ref['id'] - db.instance_set_state(context, id, power_state.RUNNING, 'running') + db.instance_set_state(ctxt, + instance_ref['id'], + power_state.RUNNING, + 'running') for v in instance_ref['volumes']: - db.volume_update(context, + db.volume_update(ctxt, v['id'], {'status': 'in-use'}) @@ -1052,20 +1059,20 @@ class LibvirtConnection(object): timer = utils.LoopingCall(f=None) def wait_for_live_migration(): - + """waiting for live migration completion""" try: - state = self.get_info(instance_ref.name)['state'] + self.get_info(instance_ref.name)['state'] except exception.NotFound: timer.stop() - self._post_live_migration(context, instance_ref, dest) + self._post_live_migration(ctxt, instance_ref, dest) timer.f = wait_for_live_migration timer.start(interval=0.5, now=True) - def _post_live_migration(self, context, instance_ref, dest): + def _post_live_migration(self, ctxt, instance_ref, dest): """ - Post operations for live migration. - Mainly, database updating. + Post operations for live migration. + Mainly, database updating. """ LOG.info('post livemigration operation is started..') # Detaching volumes. @@ -1079,61 +1086,61 @@ class LibvirtConnection(object): 'nova.virt.libvirt_conn.IptablesFirewallDriver': try: self.firewall_driver.unfilter_instance(instance_ref) - except KeyError, e: + except KeyError: pass # Database updating. ec2_id = instance_ref['hostname'] instance_id = instance_ref['id'] - fixed_ip = db.instance_get_fixed_address(context, instance_id) + fixed_ip = db.instance_get_fixed_address(ctxt, instance_id) # Not return if fixed_ip is not found, otherwise, # instance never be accessible.. if None == fixed_ip: logging.warn('fixed_ip is not found for %s ' % ec2_id) - db.fixed_ip_update(context, fixed_ip, {'host': dest}) - network_ref = db.fixed_ip_get_network(context, fixed_ip) - db.network_update(context, network_ref['id'], {'host': dest}) + db.fixed_ip_update(ctxt, fixed_ip, {'host': dest}) + network_ref = db.fixed_ip_get_network(ctxt, fixed_ip) + db.network_update(ctxt, network_ref['id'], {'host': dest}) try: floating_ip \ - = db.instance_get_floating_address(context, instance_id) + = db.instance_get_floating_address(ctxt, instance_id) # Not return if floating_ip is not found, otherwise, # instance never be accessible.. if None == floating_ip: - logging.error('floating_ip is not found for %s ' % ec2_id) + LOG.info(_('floating_ip is not found for %s'), ec2_id) else: - floating_ip_ref = db.floating_ip_get_by_address(context, + floating_ip_ref = db.floating_ip_get_by_address(ctxt, floating_ip) - db.floating_ip_update(context, + db.floating_ip_update(ctxt, floating_ip_ref['address'], {'host': dest}) except exception.NotFound: - logging.debug('%s doesnt have floating_ip.. ' % ec2_id) + LOG.info(_('floating_ip is not found for %s'), ec2_id) except: - msg = 'Live migration: Unexpected error:' - msg += '%s cannot inherit floating ip.. ' % ec2_id - logging.error(_(msg)) + msg = ("""Live migration: Unexpected error:""" + """%s cannot inherit floating ip..""") + LOG.error(_(msg), ec2_id) # Restore instance/volume state - db.instance_update(context, + db.instance_update(ctxt, instance_id, {'state_description': 'running', 'state': power_state.RUNNING, 'host': dest}) for v in instance_ref['volumes']: - db.volume_update(context, + db.volume_update(ctxt, v['id'], {'status': 'in-use'}) - logging.info(_('Live migrating %s to %s finishes successfully') + LOG.info(_('Live migrating %s to %s finishes successfully') % (ec2_id, dest)) msg = _(("""Known error: the below error is nomally occurs.\n""" """Just check if iinstance is successfully migrated.\n""" """libvir: QEMU error : Domain not found: no domain """ """with matching name..""")) - logging.info(msg) + LOG.info(msg) class FirewallDriver(object): diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index c10f73fe7..1e7933f51 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -215,15 +215,15 @@ class XenAPIConnection(object): def get_vcpu_number(self): """This method is supported only libvirt. """ - return -1 + return def get_memory_mb(self): """This method is supported only libvirt..""" - return -1 + return def get_local_gb(self): """This method is supported only libvirt..""" - return -1 + return def get_hypervisor_type(self): """This method is supported only libvirt..""" @@ -231,12 +231,18 @@ class XenAPIConnection(object): def get_hypervisor_version(self): """This method is supported only libvirt..""" - return -1 + return def compare_cpu(self, xml): + """This method is supported only libvirt..""" + raise NotImplementedError('This method is supported only libvirt.') + + def ensure_filtering_rules_for_instance(self, instance_ref): + """This method is supported only libvirt..""" raise NotImplementedError('This method is supported only libvirt.') def live_migration(self, context, instance_ref, dest): + """This method is supported only libvirt..""" raise NotImplementedError('This method is supported only libvirt.') diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 1735d79eb..906eb86ea 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -153,6 +153,6 @@ class VolumeManager(manager.Manager): def check_for_export(self, context, instance_id): """Make sure whether volume is exported.""" if FLAGS.volume_driver == 'nova.volume.driver.AOEDriver': - instance_ref = self.db.instance_get(instance_id) + instance_ref = self.db.instance_get(context, instance_id) for v in instance_ref['volumes']: self.driver.check_for_export(context, v['id']) -- cgit From 09f2c4729456443c4874a8cadc53299817d6371a Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Mon, 31 Jan 2011 18:41:10 +0900 Subject: 1. Discard nova-manage host list Reason: nova-manage service list can be replacement. Changes: nova-manage 2. Fix resource checking inappropriate design. Reason: nova.scheduler.driver.has_enough_resource has inappropriate design, so fix it. This method didnt check free memory but check total memory. We need to register free memory onto databases(periodically). But periodically updating may causes flooding request to db in case of many compute-node. Currently, since memory information is only used in this feature, we take the choice that administrators manually has to execute nova-manage to let compute node update their own memory information. Changes: nova.db.sqlalchemy.models - Adding memory_mb_used, local_gb_used, vcpu_used column to Service. (local_gb and vcpu is just for reference to admins for now) nova.compute.manager - Changing nova.compute.manager.update_service Service table column is changed, so updating method must be changed. - Adding nova.compute.manager.update_available_resource a responder to admin's request to let compute nodes update their memory infomation nova.virt.libvirt_conn nova.virt.xenapi_conn nova.virt.fake - Adding getter method for memory_mb_used/local_gb_used/vcpu_used. nova-manage - request method to let compute nodes update their own memory info. --- nova/compute/manager.py | 54 ++++++++++++- nova/db/sqlalchemy/models.py | 5 +- nova/rpc.py | 3 + nova/scheduler/driver.py | 67 +++++++++++------ nova/scheduler/manager.py | 11 ++- nova/tests/test_compute.py | 55 ++++++++++++-- nova/tests/test_scheduler.py | 175 ++++++++++++++++++++++++++----------------- nova/tests/test_virt.py | 86 ++++++++++++++++++++- nova/utils.py | 18 +++++ nova/virt/fake.py | 22 ++++-- nova/virt/libvirt_conn.py | 37 +++++++-- nova/virt/xenapi_conn.py | 22 ++++-- 12 files changed, 429 insertions(+), 126 deletions(-) (limited to 'nova') diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 4acba7153..e3c5d24b6 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -134,9 +134,12 @@ class ComputeManager(manager.Manager): raise exception.Invalid(msg) # Updating host information - vcpu = self.driver.get_vcpu_number() - memory_mb = self.driver.get_memory_mb() - local_gb = self.driver.get_local_gb() + vcpu = self.driver.get_vcpu_total() + memory_mb = self.driver.get_memory_mb_total() + local_gb = self.driver.get_local_gb_total() + vcpu_u = self.driver.get_vcpu_used() + memory_mb_u = self.driver.get_memory_mb_used() + local_gb_u = self.driver.get_local_gb_used() hypervisor = self.driver.get_hypervisor_type() version = self.driver.get_hypervisor_version() cpu_info = self.driver.get_cpu_info() @@ -146,10 +149,42 @@ class ComputeManager(manager.Manager): {'vcpus': vcpu, 'memory_mb': memory_mb, 'local_gb': local_gb, + 'vcpus_used':vcpu_u, + 'memory_mb_used': memory_mb_u, + 'local_gb_used': local_gb_u, 'hypervisor_type': hypervisor, 'hypervisor_version': version, 'cpu_info': cpu_info}) + def update_available_resource(self, context): + """ + update compute node specific info to DB. + Alghough this might be subset of update_service, + udpate_service() is used only nova-compute is lauched. + On the other hand, this method is used whenever administrators + request comes. + """ + try: + service_ref = self.db.service_get_by_args(context, + self.host, + 'nova-compute') + except exception.NotFound: + msg = _(("""Cannot update resource info.""" + """ Because no service record found.""")) + raise exception.Invalid(msg) + + # Updating host information + vcpu_u = self.driver.get_vcpu_used() + memory_mb_u = self.driver.get_memory_mb_used() + local_gb_u = self.driver.get_local_gb_used() + + self.db.service_update(context, + service_ref['id'], + {'vcpus_used':vcpu_u, + 'memory_mb_used': memory_mb_u, + 'local_gb_used': local_gb_u}) + return + def _update_state(self, context, instance_id): """Update the state of an instance from the driver info.""" # FIXME(ja): include other fields from state? @@ -596,6 +631,19 @@ class ComputeManager(manager.Manager): """ Check the host cpu is compatible to a cpu given by xml.""" return self.driver.compare_cpu(cpu_info) + def mktmpfile(self, context): + """make tmpfile under FLAGS.instance_path.""" + return utils.mktmpfile(FLAGS.instances_path) + + def exists(self, context, path): + """Confirm existence of the tmpfile given by path.""" + if not utils.exists(path): + raise exception.NotFound(_('%s not found') % path) + + def remove(self, context, path): + """remove the tmpfile given by path.""" + return utils.remove(path) + def pre_live_migration(self, context, instance_id): """Any preparation for live migration at dst host.""" diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 7c40d5596..217b14bf7 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -164,7 +164,10 @@ class Service(BASE, NovaBase): vcpus = Column(Integer, nullable=True) memory_mb = Column(Integer, nullable=True) local_gb = Column(Integer, nullable=True) - hypervisor_type = Column(String(128), nullable=True) + vcpus_used = Column(Integer, nullable=True) + memory_mb_used = Column(Integer, nullable=True) + local_gb_used = Column(Integer, nullable=True) + hypervisor_type = Column(Text(), nullable=True) hypervisor_version = Column(Integer, nullable=True) # Note(masumotok): Expected Strings example: # diff --git a/nova/rpc.py b/nova/rpc.py index 49b11602b..cf4004079 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -250,6 +250,9 @@ def msg_reply(msg_id, reply=None, failure=None): try: publisher.send({'result': reply, 'failure': failure}) except TypeError: + print '>>>>>>>>>>>>>>>>>>' + print reply + print '>>>>>>>>>>>>>>>>>>' publisher.send( {'result': dict((k, repr(v)) for k, v in reply.__dict__.iteritems()), diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index d4ad42388..937f09c6f 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -33,6 +33,7 @@ from nova.compute import power_state FLAGS = flags.FLAGS flags.DEFINE_integer('service_down_time', 60, 'maximum time since last checkin for up service') +flags.DECLARE('instances_path', 'nova.compute.manager') class NoValidHost(exception.Error): @@ -163,6 +164,8 @@ class Scheduler(object): http://wiki.libvirt.org/page/TodoPreMigrationChecks """ + # Checking shared storage connectivity + self.mounted_on_same_shared_storage(context, instance_ref, dest) # Checking dest exists. dservice_refs = db.service_get_all_by_host(context, dest) @@ -207,38 +210,60 @@ class Scheduler(object): raise e def has_enough_resource(self, context, instance_ref, dest): - """Check if destination host has enough resource for live migration""" + """ + Check if destination host has enough resource for live migration. + Currently, only memory checking has been done. + If storage migration(block migration, meaning live-migration + without any shared storage) will be available, local storage + checking is also necessary. + """ # Getting instance information ec2_id = instance_ref['hostname'] - vcpus = instance_ref['vcpus'] mem = instance_ref['memory_mb'] - hdd = instance_ref['local_gb'] - # Gettin host information + # Getting host information service_refs = db.service_get_all_by_host(context, dest) if len(service_refs) <= 0: raise exception.Invalid(_('%s does not exists.') % dest) service_ref = service_refs[0] - total_cpu = int(service_ref['vcpus']) - total_mem = int(service_ref['memory_mb']) - total_hdd = int(service_ref['local_gb']) + mem_total = int(service_ref['memory_mb']) + mem_used = int(service_ref['memory_mb_used']) + mem_avail = mem_total - mem_used + mem_inst = instance_ref['memory_mb'] + if mem_avail <= mem_inst: + msg = _('%s is not capable to migrate %s(host:%s <= instance:%s)') + raise exception.NotEmpty(msg % (dest, ec2_id, mem_avail, mem_inst)) - instances_refs = db.instance_get_all_by_host(context, dest) - for i_ref in instances_refs: - total_cpu -= int(i_ref['vcpus']) - total_mem -= int(i_ref['memory_mb']) - total_hdd -= int(i_ref['local_gb']) + def mounted_on_same_shared_storage(self, context, instance_ref, dest): + """ + Check if /nova-inst-dir/insntances is mounted same storage at + live-migration src and dest host. + """ + src = instance_ref['host'] + dst_t = db.queue_get_for(context, FLAGS.compute_topic, dest) + src_t = db.queue_get_for(context, FLAGS.compute_topic, src) - # Checking host has enough information - logging.debug(_('host(%s) remains vcpu:%s mem:%s hdd:%s,') % - (dest, total_cpu, total_mem, total_hdd)) - logging.debug(_('instance(%s) has vcpu:%s mem:%s hdd:%s,') % - (ec2_id, vcpus, mem, hdd)) + # create tmpfile at dest host + try: + filename = rpc.call(context, dst_t, {"method": 'mktmpfile'}) + except rpc.RemoteError, e: + msg = _("Cannot create tmpfile at %s to confirm shared storage.") + logging.error(msg % FLAGS.instance_path) + raise e - if total_cpu <= vcpus or total_mem <= mem or total_hdd <= hdd: - raise exception.NotEmpty(_('%s is not capable to migrate %s') % - (dest, ec2_id)) + # make sure existence at src host. + try: + rpc.call(context, src_t, + {"method": 'exists', "args":{'path':filename}}) + + except (rpc.RemoteError, exception.NotFound), e: + msg = (_("""Cannot comfirm %s at %s to confirm shared storage.""" + """Check if %s is same shared storage""")) + logging.error(msg % FLAGS.instance_path) + raise e - logging.debug(_('%s has_enough_resource() for %s') % (dest, ec2_id)) + # then remove. + rpc.call(context, dst_t, + {"method": 'remove', "args":{'path':filename}}) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index a181225a6..b40f46a85 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -84,7 +84,10 @@ class SchedulerManager(manager.Manager): # Getting physical resource information h_resource = {'vcpus': service_ref['vcpus'], 'memory_mb': service_ref['memory_mb'], - 'local_gb': service_ref['local_gb']} + 'local_gb': service_ref['local_gb'], + 'vcpus_used': service_ref['vcpus_used'], + 'memory_mb_used': service_ref['memory_mb_used'], + 'local_gb_used': service_ref['local_gb_used']} # Getting usage resource information u_resource = {} @@ -108,8 +111,8 @@ class SchedulerManager(manager.Manager): hdd = db.instance_get_disk_sum_by_host_and_project(context, host, p_id) - u_resource[p_id] = {'vcpus': vcpus, - 'memory_mb': mem, - 'local_gb': hdd} + u_resource[p_id] = {'vcpus': int(vcpus), + 'memory_mb': int(mem), + 'local_gb': int(hdd)} return {'ret': True, 'phy_resource': h_resource, 'usage': u_resource} diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 344c2d2b5..8d3ac315d 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -268,7 +268,8 @@ class ComputeTestCase(test.TestCase): """ def dic_key_check(dic): - validkey = ['vcpus', 'memory_mb', 'local_gb', + validkey = ['vcpus', 'memory_mb', 'local_gb', + 'vcpus_used', 'memory_mb_used', 'local_gb_used', 'hypervisor_type', 'hypervisor_version', 'cpu_info'] return (list(set(validkey)) == list(set(dic.keys()))) @@ -286,13 +287,55 @@ class ComputeTestCase(test.TestCase): self.compute.db = dbmock self.mox.ReplayAll() + self.compute.update_service('dummy', host, binary) + self.mox.ResetAll() + + def test_update_available_resource_exception(self): + """a testcase of update_available_resource raises exception""" + host = 'foo' + binary = 'nova-compute' + ctxt = context.get_admin_context() + dbmock = self.mox.CreateMock(db) + dbmock.service_get_by_args(mox.IgnoreArg(), + mox.StrContains(host), + mox.StrContains(binary)).\ + AndRaise(exception.NotFound()) + self.compute.db = dbmock + self.compute.host = host + self.mox.ReplayAll() try: - self.compute.update_service('dummy', host, binary) + self.compute.update_available_resource(ctxt) except exception.Invalid, e: - msg = 'Cannot insert compute manager specific info' + msg = 'Cannot update resource info.' c1 = ( 0 <= e.message.find(msg)) self.assertTrue(c1) - self.mox.ResetAll() + self.mox.UnsetStubs() + + def test_update_available_resource_success(self): + """a testcase of update_available_resource finishes with no errors""" + + def dic_key_check(dic): + validkey = [ 'vcpus_avail', 'memory_mb_avail', 'local_gb_avail'] + return (list(set(validkey)) == list(set(dic.keys()))) + + host = 'foo' + binary = 'nova-compute' + ctxt = context.get_admin_context() + service_ref = {'id':1, 'binary':'nova-compute', 'topic':'compute'} + dbmock = self.mox.CreateMock(db) + dbmock.service_get_by_args(mox.IgnoreArg(), + mox.StrContains(host), + mox.StrContains(binary)).\ + AndReturn(service_ref) + dbmock.service_update(mox.IgnoreArg(), + service_ref['id'], + mox.Func(dic_key_check)) + + self.compute.db = dbmock + self.compute.host = host + self.mox.ReplayAll() + self.compute.update_available_resource(ctxt) + self.mox.UnsetStubs() def _setup_other_managers(self): self.volume_manager = utils.import_object(FLAGS.volume_manager) @@ -444,7 +487,7 @@ class ComputeTestCase(test.TestCase): rpc.call(c, db.queue_get_for(c, FLAGS.compute_topic, dest), {"method": "pre_live_migration", "args": {'instance_id': i_id}}).\ - InAnyOrder('g1').AndRaise(rpc.RemoteError('du', 'mm', 'y')) + InAnyOrder('g1').AndRaise(rpc.RemoteError('', '', '')) self.mox.StubOutWithMock(compute_manager.LOG, 'error') compute_manager.LOG.error('Pre live migration for %s failed at %s', instance_ref['hostname'], dest) @@ -480,7 +523,7 @@ class ComputeTestCase(test.TestCase): rpc.call(c, compute_topic, {"method": "pre_live_migration", "args": {'instance_id': i_id}}).\ - AndRaise(rpc.RemoteError('du', 'mm', 'y')) + AndRaise(rpc.RemoteError('', '', '')) self.mox.StubOutWithMock(compute_manager.LOG, 'error') compute_manager.LOG.error('Pre live migration for %s failed at %s', instance_ref['hostname'], dest) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index c62bca9b1..36d99d666 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -41,6 +41,7 @@ from nova.db.sqlalchemy import models FLAGS = flags.FLAGS flags.DECLARE('max_cores', 'nova.scheduler.simple') flags.DECLARE('stub_network', 'nova.compute.manager') +flags.DECLARE('instances_path', 'nova.compute.manager') class TestDriver(driver.Scheduler): @@ -111,7 +112,8 @@ class SchedulerTestCase(test.TestCase): scheduler = manager.SchedulerManager() dest = 'dummydest' ctxt = context.get_admin_context() - r0 = {'vcpus':16, 'memory_mb':32, 'local_gb':100} + r0 = {'vcpus':16, 'memory_mb':32, 'local_gb':100, + 'vcpus_used':16, 'memory_mb_used':32, 'local_gb_used':10} service_ref = {'id':1, 'host':dest} service_ref.update(r0) @@ -140,7 +142,8 @@ class SchedulerTestCase(test.TestCase): scheduler = manager.SchedulerManager() dest = 'dummydest' ctxt = context.get_admin_context() - r0 = {'vcpus':16, 'memory_mb':32, 'local_gb':100} + r0 = {'vcpus':16, 'memory_mb':32, 'local_gb':100, + 'vcpus_used':16, 'memory_mb_used':32, 'local_gb_used':10} r1 = {'vcpus':10, 'memory_mb':4, 'local_gb':20} r2 = {'vcpus':10, 'memory_mb':20, 'local_gb':30} service_ref = {'id':1, 'host':dest} @@ -148,7 +151,7 @@ class SchedulerTestCase(test.TestCase): instance_ref2 = {'id':2, 'project_id':'p-01', 'host':'dummy'} instance_ref2.update(r1) instance_ref3 = {'id':3, 'project_id':'p-02', 'host':'dummy'} - instance_ref3.update(r1) + instance_ref3.update(r2) self.mox.StubOutWithMock(manager, 'db', use_mock_anything=True) manager.db.service_get_all_compute_sorted(mox.IgnoreArg()).\ @@ -176,6 +179,7 @@ class SchedulerTestCase(test.TestCase): self.assertTrue( c1 and c2 and c3 and c4 and c5 and c6) self.mox.UnsetStubs() + class ZoneSchedulerTestCase(test.TestCase): """Test case for zone scheduler""" def setUp(self): @@ -495,7 +499,7 @@ class SimpleDriverTestCase(test.TestCase): ctxt = context.get_admin_context() topic = FLAGS.compute_topic i_ref = {'id':1, 'hostname':'i-00000001', 'host':'dummy', - 'volumes':[{'id':1}, {'id':2}]} + 'volumes':[{'id':1}, {'id':2}]} dest = 'dummydest' self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) @@ -793,7 +797,10 @@ class SimpleDriverTestCase(test.TestCase): ctxt = context.get_admin_context() topic = FLAGS.compute_topic i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + driver_i = self.scheduler.driver + self.mox.StubOutWithMock(driver_i, 'mounted_on_same_shared_storage') + driver_i.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([]) @@ -813,6 +820,7 @@ class SimpleDriverTestCase(test.TestCase): Original host(an instance launched on) does not exist. """ dest = 'dummydest' + driver_i = self.scheduler.driver ctxt = context.get_admin_context() topic = FLAGS.compute_topic i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', 'launched_on':'h1'} @@ -821,6 +829,8 @@ class SimpleDriverTestCase(test.TestCase): service_ref.__setitem__('topic', 'compute') service_ref.__setitem__('host', i_ref['host']) + self.mox.StubOutWithMock(driver_i, 'mounted_on_same_shared_storage') + driver_i.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([service_ref]) @@ -844,6 +854,7 @@ class SimpleDriverTestCase(test.TestCase): Original host and dest host has different hypervisor type. """ dest = 'dummydest' + driver_i = self.scheduler.driver ctxt = context.get_admin_context() topic = FLAGS.compute_topic i_ref = {'id':1, 'hostname':'i-01', @@ -856,6 +867,8 @@ class SimpleDriverTestCase(test.TestCase): service_ref2.__setitem__('id', 2) service_ref2.__setitem__('hypervisor_type', 'xen') + self.mox.StubOutWithMock(driver_i, 'mounted_on_same_shared_storage') + driver_i.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([service_ref]) @@ -879,6 +892,7 @@ class SimpleDriverTestCase(test.TestCase): Original host and dest host has different hypervisor version. """ dest = 'dummydest' + driver_i = self.scheduler.driver ctxt = context.get_admin_context() topic = FLAGS.compute_topic i_ref = {'id':1, 'hostname':'i-01', @@ -891,6 +905,8 @@ class SimpleDriverTestCase(test.TestCase): service_ref2.__setitem__('id', 2) service_ref2.__setitem__('hypervisor_version', 12001) + self.mox.StubOutWithMock(driver_i, 'mounted_on_same_shared_storage') + driver_i.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([service_ref]) @@ -914,6 +930,7 @@ class SimpleDriverTestCase(test.TestCase): Original host and dest host has different hypervisor version. """ dest = 'dummydest' + driver_i = self.scheduler.driver ctxt = context.get_admin_context() topic = FLAGS.compute_topic i_ref = {'id':1, 'hostname':'i-01', @@ -927,6 +944,8 @@ class SimpleDriverTestCase(test.TestCase): service_ref2.__setitem__('hypervisor_version', 12000) service_ref2.__setitem__('cpuinfo', 'info') + self.mox.StubOutWithMock(driver_i, 'mounted_on_same_shared_storage') + driver_i.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([service_ref]) @@ -956,6 +975,7 @@ class SimpleDriverTestCase(test.TestCase): The testcase make sure everything finished with no error. """ dest = 'dummydest' + driver_i = self.scheduler.driver ctxt = context.get_admin_context() topic = FLAGS.compute_topic i_ref = {'id':1, 'hostname':'i-01', @@ -969,6 +989,8 @@ class SimpleDriverTestCase(test.TestCase): service_ref2.__setitem__('hypervisor_version', 12000) service_ref2.__setitem__('cpuinfo', 'info') + self.mox.StubOutWithMock(driver_i, 'mounted_on_same_shared_storage') + driver_i.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([service_ref]) @@ -988,121 +1010,134 @@ class SimpleDriverTestCase(test.TestCase): self.assertTrue(ret == None) self.mox.UnsetStubs() - def test_has_enough_resource_lack_resource_vcpu(self): + def test_has_enough_resource_lack_resource_memory(self): """ A testcase of driver.has_enough_resource. - Lack of vcpu.(boundary check) + Lack of memory_mb.(boundary check) """ dest = 'dummydest' ctxt = context.get_admin_context() topic = FLAGS.compute_topic - service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} - i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', - 'vcpus':6, 'memory_mb':8, 'local_gb':10} - i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} - i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} + service_ref = {'id':1, 'memory_mb':32, 'memory_mb_used':12, 'local_gb':100} + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':20, 'local_gb':10} self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([service_ref]) - driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ - AndReturn([i_ref2, i_ref3]) self.mox.ReplayAll() try: - self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) + self.scheduler.driver.has_enough_resource(ctxt, i_ref, dest) except exception.NotEmpty, e: msg = 'is not capable to migrate' self.assertTrue(e.message.find(msg) >= 0) self.mox.UnsetStubs() + self.mox.UnsetStubs() - def test_has_enough_resource_lack_resource_memory(self): + def test_has_enough_resource_works_correctly(self): """ - A testcase of driver.has_enough_resource. - Lack of memory_mb.(boundary check) + A testcase of driver.has_enough_resource + to make sure everything finished with no error. """ dest = 'dummydest' ctxt = context.get_admin_context() topic = FLAGS.compute_topic - service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} - i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', - 'vcpus':5, 'memory_mb':16, 'local_gb':10} - i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} - i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', + service_ref = {'id':1, 'memory_mb':120, 'memory_mb_used':32} + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', 'vcpus':5, 'memory_mb':8, 'local_gb':10} self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([service_ref]) - driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ - AndReturn([i_ref2, i_ref3]) self.mox.ReplayAll() - try: - self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) - except exception.NotEmpty, e: - msg = 'is not capable to migrate' - self.assertTrue(e.message.find(msg) >= 0) + ret = self.scheduler.driver.has_enough_resource(ctxt, i_ref, dest) + self.assertTrue(ret == None) self.mox.UnsetStubs() + + def test_mounted_on_same_shared_storage_cannot_make_tmpfile(self): + """ + A testcase of driver.mounted_on_same_shared_storage + checks log message when dest host cannot make tmpfile. + """ + dest = 'dummydest' + driver_i = self.scheduler.driver + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + fpath = '/test/20110127120000' + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + + self.mox.StubOutWithMock(driver, 'rpc', use_mock_anything=True) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(ctxt, FLAGS.compute_topic, dest), + {"method": 'mktmpfile'}).AndRaise(rpc.RemoteError('', '', '')) + self.mox.StubOutWithMock(driver.logging, 'error') + msg = _("Cannot create tmpfile at %s to confirm shared storage.") + driver.logging.error(msg % FLAGS.instances_path) + + self.mox.ReplayAll() + self.assertRaises(rpc.RemoteError, + driver_i.mounted_on_same_shared_storage, + ctxt, i_ref, dest) self.mox.UnsetStubs() - def test_has_enough_resource_lack_resource_disk(self): + def test_mounted_on_same_shared_storage_cannot_comfirm_tmpfile(self): """ - A testcase of driver.has_enough_resource. - Lack of local_gb.(boundary check) + A testcase of driver.mounted_on_same_shared_storage + checks log message when src host cannot comfirm tmpfile. """ - scheduler = manager.SchedulerManager() dest = 'dummydest' + driver_i = self.scheduler.driver ctxt = context.get_admin_context() topic = FLAGS.compute_topic - service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} - i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':80} - i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} - i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} + fpath = '/test/20110127120000' + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + + self.mox.StubOutWithMock(driver, 'rpc', use_mock_anything=True) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(ctxt, FLAGS.compute_topic, dest), + {"method": 'mktmpfile'}).AndReturn(fpath) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(ctxt, FLAGS.compute_topic, i_ref['host']), + {"method": 'exists', "args":{'path':fpath}}).\ + AndRaise(rpc.RemoteError('','','')) + self.mox.StubOutWithMock(driver.logging, 'error') + msg = _("Cannot create tmpfile at %s to confirm shared storage.") + driver.logging.error(msg % FLAGS.instances_path) - self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) - driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ - AndReturn([service_ref]) - driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ - AndReturn([i_ref2, i_ref3]) - self.mox.ReplayAll() - try: - self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) - except exception.NotEmpty, e: - msg = 'is not capable to migrate' - self.assertTrue(e.message.find(msg) >= 0) + self.assertRaises(rpc.RemoteError, + driver_i.mounted_on_same_shared_storage, + ctxt, i_ref, dest) self.mox.UnsetStubs() - def test_has_enough_resource_works_correctly(self): + + def test_mounted_on_same_shared_storage_works_correctly(self): """ - A testcase of driver.has_enough_resource + A testcase of driver.mounted_on_same_shared_storage to make sure everything finished with no error. """ dest = 'dummydest' ctxt = context.get_admin_context() topic = FLAGS.compute_topic - service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} - i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} - i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} - i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} + fpath = '/test/20110127120000' + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + + self.mox.StubOutWithMock(driver, 'rpc', use_mock_anything=True) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(mox.IgnoreArg(), FLAGS.compute_topic, dest), + {"method": 'mktmpfile'}).AndReturn(fpath) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(mox.IgnoreArg(), FLAGS.compute_topic, i_ref['host']), + {"method": 'exists', "args":{'path':fpath}}) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(mox.IgnoreArg(), FLAGS.compute_topic, dest), + {"method": 'remove', "args":{'path':fpath}}) - self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) - driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ - AndReturn([service_ref]) - driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ - AndReturn([i_ref2, i_ref3]) - self.mox.ReplayAll() - ret = self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) + ret = self.scheduler.driver.mounted_on_same_shared_storage(ctxt, + i_ref, + dest) self.assertTrue(ret == None) self.mox.UnsetStubs() diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 177e8f021..2828baced 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -241,11 +241,11 @@ class LibvirtConnTestCase(test.TestCase): uri = conn.get_uri() self.assertEquals(uri, testuri) - def test_get_memory_mb(self): + def test_get_vcpu_total(self): """ - Check if get_memory_mb returns memory value + Check if get_vcpu_total returns appropriate cpu value Connection/OS/driver differenct does not matter for this method, - so everyone can execute for checking. + everyone can execute for checking. """ try: self._driver_dependent_test_setup() @@ -254,9 +254,87 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) - self.assertTrue(0 < conn.get_memory_mb()) + self.assertTrue(0 < conn.get_vcpu_total()) self.mox.UnsetStubs() + + def test_get_memory_mb_total(self): + """Check if get_memory_mb returns appropriate memory value""" + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertTrue(0 < conn.get_memory_mb_total()) + self.mox.UnsetStubs() + + def test_get_local_gb_total(self): + """Check if get_local_gb_total returns appropriate disk value""" + # Note(masumotok): cannot test b/c FLAGS.instances_path is + # inevitable for this test.. + #try: + # self._driver_dependent_test_setup() + #except: + # return + # + #self.mox.ReplayAll() + #conn = libvirt_conn.LibvirtConnection(False) + #self.assertTrue(0 < conn.get_local_gb_total()) + #self.mox.UnsetStubs() + pass + + def test_get_vcpu_used(self): + """Check if get_local_gb_total returns appropriate disk value""" + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', use_mock_anything=True) + libvirt_conn.LibvirtConnection._conn.listDomainsID().AndReturn([1,2]) + vdmock = self.mox.CreateMock(libvirt.virDomain) + self.mox.StubOutWithMock(vdmock, "vcpus", use_mock_anything=True) + vdmock.vcpus().AndReturn(['', [('dummycpu'), ('dummycpu')]]) + vdmock.vcpus().AndReturn(['', [('dummycpu'), ('dummycpu')]]) + libvirt_conn.LibvirtConnection._conn.lookupByID(mox.IgnoreArg()).\ + AndReturn(vdmock) + libvirt_conn.LibvirtConnection._conn.lookupByID(mox.IgnoreArg()).\ + AndReturn(vdmock) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertTrue( conn.get_vcpu_used() == 4) + self.mox.UnsetStubs() + + def test_get_memory_mb_used(self): + """Check if get_memory_mb returns appropriate memory value""" + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertTrue(0 < conn.get_memory_mb_used()) + self.mox.UnsetStubs() + + def test_get_local_gb_used(self): + """Check if get_local_gb_total returns appropriate disk value""" + # Note(masumotok): cannot test b/c FLAGS.instances_path is + # inevitable for this test.. + #try: + # self._driver_dependent_test_setup() + #except: + # return + + #self.mox.ReplayAll() + #conn = libvirt_conn.LibvirtConnection(False) + #self.assertTrue(0 < conn.get_local_gb_used()) + #self.mox.UnsetStubs() + pass + def test_get_cpu_info_works_correctly(self): """ Check if get_cpu_info works correctly. diff --git a/nova/utils.py b/nova/utils.py index 6d3ddd092..e8919d7a5 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -303,6 +303,24 @@ def str_dict_replace(s, mapping): return s +def mktmpfile(dir): + """create tmpfile under dir, and return filename.""" + filename = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S') + fpath = os.path.join(dir, filename) + open(fpath, 'a+').write(fpath+'\n') + return fpath + + +def exists(filename): + """check file path existence.""" + return os.path.exists(filename) + + +def remove(filename): + """remove file.""" + return os.remove(filename) + + class LazyPluggable(object): """A pluggable backend loaded lazily based on some value.""" diff --git a/nova/virt/fake.py b/nova/virt/fake.py index f469af681..4bf477f5b 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -314,16 +314,28 @@ class FakeConnection(object): """This method is supported only libvirt. """ return - def get_vcpu_number(self): + def get_vcpu_total(self): """This method is supported only libvirt. """ return - def get_memory_mb(self): - """This method is supported only libvirt..""" + def get_memory_mb_total(self): + """This method is supported only libvirt. """ return - def get_local_gb(self): - """This method is supported only libvirt..""" + def get_local_gb_total(self): + """This method is supported only libvirt. """ + return + + def get_vcpu_used(self): + """This method is supported only libvirt. """ + return + + def get_memory_mb_used(self): + """This method is supported only libvirt. """ + return + + def get_local_gb_used(self): + """This method is supported only libvirt. """ return def get_hypervisor_type(self): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 49dd03c57..aefa32dcb 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -835,21 +835,44 @@ class LibvirtConnection(object): return interfaces - def get_vcpu_number(self): + def get_vcpu_total(self): """ Get vcpu number of physical computer. """ - return self._conn.getMaxVcpus(None) + return open('/proc/cpuinfo').read().count('processor') - def get_memory_mb(self): - """Get the memory size of physical computer .""" + def get_memory_mb_total(self): + """Get the total memory size(MB) of physical computer .""" meminfo = open('/proc/meminfo').read().split() idx = meminfo.index('MemTotal:') # transforming kb to mb. return int(meminfo[idx + 1]) / 1024 - def get_local_gb(self): - """Get the hdd size of physical computer .""" + def get_local_gb_total(self): + """Get the total hdd size(GB) of physical computer .""" hddinfo = os.statvfs(FLAGS.instances_path) - return hddinfo.f_bsize * hddinfo.f_blocks / 1024 / 1024 / 1024 + return hddinfo.f_frsize * hddinfo.f_blocks / 1024 / 1024 / 1024 + + def get_vcpu_used(self): + """ Get vcpu available number of physical computer. """ + total = 0 + for i in self._conn.listDomainsID(): + dom = self._conn.lookupByID(i) + total += len(dom.vcpus()[1]) + return total + + def get_memory_mb_used(self): + """Get the free memory size(MB) of physical computer.""" + m = open('/proc/meminfo').read().split() + idx1 = m.index('MemFree:') + idx2 = m.index('Buffers:') + idx3 = m.index('Cached:') + avail = (int(m[idx1+1]) + int(m[idx2+1]) + int(m[idx3+1])) / 1024 + return self.get_memory_mb_total() - avail + + def get_local_gb_used(self): + """Get the free hdd size(GB) of physical computer .""" + hddinfo = os.statvfs(FLAGS.instances_path) + avail = hddinfo.f_frsize * hddinfo.f_bavail / 1024 / 1024 / 1024 + return self.get_local_gb_total() - avail def get_hypervisor_type(self): """ Get hypervisor type """ diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 1e7933f51..902879d09 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -213,16 +213,28 @@ class XenAPIConnection(object): """This method is supported only libvirt. """ return - def get_vcpu_number(self): + def get_vcpu_total(self): """This method is supported only libvirt. """ return - def get_memory_mb(self): - """This method is supported only libvirt..""" + def get_memory_mb_total(self): + """This method is supported only libvirt. """ return - def get_local_gb(self): - """This method is supported only libvirt..""" + def get_local_gb_total(self): + """This method is supported only libvirt. """ + return + + def get_vcpu_used(self): + """This method is supported only libvirt. """ + return + + def get_memory_mb_used(self): + """This method is supported only libvirt. """ + return + + def get_local_gb_used(self): + """This method is supported only libvirt. """ return def get_hypervisor_type(self): -- cgit From 0e3c86dcdc49890eecaa2d1ea64c0012e569682f Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 17 Feb 2011 22:07:00 +0100 Subject: Use a semaphore to ensure we don't run more than one iptables-restore at a time. --- nova/virt/libvirt_conn.py | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 4e0fd106f..7548fff63 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -46,6 +46,7 @@ from xml.dom import minidom from eventlet import greenthread from eventlet import event +from eventlet import semaphore from eventlet import tpool import IPy @@ -63,6 +64,7 @@ from nova.compute import power_state from nova.virt import disk from nova.virt import images +libvirt_semaphore = semaphore.Semaphore() libvirt = None libxml2 = None Template = None @@ -1237,17 +1239,19 @@ class IptablesFirewallDriver(FirewallDriver): self.apply_ruleset() def apply_ruleset(self): - current_filter, _ = self.execute('sudo iptables-save -t filter') - current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines, 4) - self.execute('sudo iptables-restore', - process_input='\n'.join(new_filter)) - if(FLAGS.use_ipv6): - current_filter, _ = self.execute('sudo ip6tables-save -t filter') + with libvirt_semaphore: + current_filter, _ = self.execute('sudo iptables-save -t filter') current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines, 6) - self.execute('sudo ip6tables-restore', + new_filter = self.modify_rules(current_lines, 4) + self.execute('sudo iptables-restore', process_input='\n'.join(new_filter)) + if(FLAGS.use_ipv6): + current_filter, _ = self.execute('sudo ip6tables-save ' + '-t filter') + current_lines = current_filter.split('\n') + new_filter = self.modify_rules(current_lines, 6) + self.execute('sudo ip6tables-restore', + process_input='\n'.join(new_filter)) def modify_rules(self, current_lines, ip_version=4): ctxt = context.get_admin_context() -- cgit From d88d74c9a0a28e0ebd6cedf694753b9ee9decdac Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Fri, 18 Feb 2011 14:15:04 +0900 Subject: fixed based on reviewer's comment. 1. erase wrapper function(remove/exists/mktempfile) from nova.utils. 2. nova-manage service describeresource(->describe_resource) 3. nova-manage service updateresource(->update_resource) 4. erase "my mistake print" statement Additional changes are made at: 1. nova.image.s3.show 2. nova.compute.api.create that's because instances cannot launched without this changes. --- nova/compute/api.py | 4 ++-- nova/compute/manager.py | 15 ++++++++++----- nova/db/sqlalchemy/migrate_repo/versions/002_bexar.py | 9 --------- nova/image/s3.py | 2 +- nova/scheduler/manager.py | 1 - nova/utils.py | 18 ------------------ nova/virt/disk.py | 1 - nova/virt/libvirt_conn.py | 2 +- 8 files changed, 14 insertions(+), 38 deletions(-) (limited to 'nova') diff --git a/nova/compute/api.py b/nova/compute/api.py index ac02dbcfa..740dd3935 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -103,9 +103,9 @@ class API(base.Base): if not is_vpn: image = self.image_service.show(context, image_id) if kernel_id is None: - kernel_id = image.get('kernelId', None) + kernel_id = image.get('kernel_id', None) if ramdisk_id is None: - ramdisk_id = image.get('ramdiskId', None) + ramdisk_id = image.get('ramdisk_id', None) # No kernel and ramdisk for raw images if kernel_id == str(FLAGS.null_kernel): kernel_id = None diff --git a/nova/compute/manager.py b/nova/compute/manager.py index cae95dd93..47dd5fd5e 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -38,6 +38,8 @@ import datetime import random import string import socket +import os +import tempfile import time import functools @@ -577,14 +579,17 @@ class ComputeManager(manager.Manager): @exception.wrap_exception def mktmpfile(self, context): """make tmpfile under FLAGS.instance_path.""" - return utils.mktmpfile(FLAGS.instances_path) + fd, name = tempfile.mkstemp(dir=FLAGS.instances_path) + # No essential reason to write dateinfo. just for debugging reason. + os.fdopen(fd, 'w').write(str(datetime.datetime.utcnow())) + return name @exception.wrap_exception def confirm_tmpfile(self, context, path): """Confirm existence of the tmpfile given by path.""" - if not utils.exists(path): + if not os.path.exists(path): raise exception.NotFound(_('%s not found') % path) - return utils.remove(path) + return os.remove(path) @exception.wrap_exception def update_available_resource(self, context): @@ -683,7 +688,7 @@ class ComputeManager(manager.Manager): Post operations for live migration. Mainly, database updating. """ - LOG.info('post_live_migration() is started..') + LOG.info(_('post_live_migration() is started..')) instance_id = instance_ref['id'] # Detaching volumes. @@ -705,7 +710,7 @@ class ComputeManager(manager.Manager): # Not return if fixed_ip is not found, otherwise, # instance never be accessible.. if None == fixed_ip: - logging.warn('fixed_ip is not found for %s ' % i_name) + LOG.warn(_('fixed_ip is not found for %s.') % i_name) self.db.fixed_ip_update(ctxt, fixed_ip, {'host': dest}) try: diff --git a/nova/db/sqlalchemy/migrate_repo/versions/002_bexar.py b/nova/db/sqlalchemy/migrate_repo/versions/002_bexar.py index 38210db85..699b837f8 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/002_bexar.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/002_bexar.py @@ -229,12 +229,3 @@ def upgrade(migrate_engine): networks.create_column(networks_cidr_v6) networks.create_column(networks_ra_server) services.create_column(services_availability_zone) - #services.create_column(services_vcpus) - #services.create_column(services_memory_mb) - #services.create_column(services_local_gb) - #services.create_column(services_vcpus_used) - #services.create_column(services_memory_mb_used) - #services.create_column(services_local_gb_used) - #services.create_column(services_hypervisor_type) - #services.create_column(services_hypervisor_version) - #services.create_column(services_cpu_info) diff --git a/nova/image/s3.py b/nova/image/s3.py index 71304cdd6..14135a1ee 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -94,7 +94,7 @@ class S3ImageService(service.BaseImageService): if FLAGS.connection_type == 'fake': return {'imageId': 'bar'} result = self.index(context) - result = [i for i in result if i['imageId'] == image_id] + result = [i for i in result if i['id'] == image_id] if not result: raise exception.NotFound(_('Image %s could not be found') % image_id) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index af54c72be..ea7ae7bd2 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -60,7 +60,6 @@ class SchedulerManager(manager.Manager): host = getattr(self.driver, driver_method)(elevated, *args, **kwargs) except AttributeError, e: - print 'manager.attrerr', e host = self.driver.schedule(elevated, topic, *args, **kwargs) rpc.cast(context, diff --git a/nova/utils.py b/nova/utils.py index 966dde667..8d7ff1f64 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -337,24 +337,6 @@ def str_dict_replace(s, mapping): return s -def mktmpfile(dir): - """create tmpfile under dir, and return filename.""" - filename = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S') - fpath = os.path.join(dir, filename) - open(fpath, 'a+').write(fpath + '\n') - return fpath - - -def exists(filename): - """check file path existence.""" - return os.path.exists(filename) - - -def remove(filename): - """remove file.""" - return os.remove(filename) - - class LazyPluggable(object): """A pluggable backend loaded lazily based on some value.""" diff --git a/nova/virt/disk.py b/nova/virt/disk.py index ec4acc452..c5565abfa 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -112,7 +112,6 @@ def inject_data(image, key=None, net=None, partition=None, nbd=False): def _link_device(image, nbd): """Link image to device using loopback or nbd""" - print '_link_device:0:', nbd, '::', image if nbd: device = _allocate_device() utils.execute('sudo qemu-nbd -c %s %s' % (device, image)) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 9b7a9ddbe..579c4593e 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -977,7 +977,7 @@ class LibvirtConnection(object): """ Update compute manager resource info on Service table. This method is called when nova-coompute launches, and - whenever admin executes "nova-manage service updateresource". + whenever admin executes "nova-manage service update_resource". """ try: -- cgit From 205810c3da4652fd0f5203f53299cd998ac7cf82 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Fri, 18 Feb 2011 11:44:06 +0100 Subject: added functionality to list only fixed ip addresses of one node and added exception handling to list method # nova-manage fixed list XXXX network IP address MAC address hostname host 10.xx.xx.0/24 10.xx.xx.5 02:16:3e:3f:33:b6 i-00000547 XXXX 10.xx.xx.0/24 10.xx.xx.9 02:16:3e:14:03:d6 i-00000548 XXXX 10.xx.xx.0/24 10.xx.xx.12 02:16:3e:20:1b:e7 i-00000549 XXXX --- nova/db/api.py | 5 +++++ nova/db/sqlalchemy/api.py | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+) (limited to 'nova') diff --git a/nova/db/api.py b/nova/db/api.py index d7f3746d2..6053c0352 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -293,6 +293,11 @@ def fixed_ip_get_all(context): return IMPL.fixed_ip_get_all(context) +def fixed_ip_get_all_by_host(context, host): + """Get all defined fixed ips used by a host.""" + return IMPL.fixed_ip_get_all_by_host(context, host) + + def fixed_ip_get_by_address(context, address): """Get a fixed ip by address or raise if it does not exist.""" return IMPL.fixed_ip_get_by_address(context, address) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 2697fac73..d07c53759 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -594,6 +594,28 @@ def fixed_ip_get_all(context, session=None): return result +@require_admin_context +def fixed_ip_get_all_by_host(context, host=None): + session = get_session() + + # FIXME: I'm sure that SQLAlchemy can handle this in a nicer way + instances = session.query(models.Instance).\ + filter_by(state=1).\ + filter_by(host=host).\ + all() + + result = [] + for instance in instances: + result.append(session.query(models.FixedIp).\ + filter_by(instance_id=instance['id']).\ + first()) + + if not result: + raise exception.NotFound(_('No fixed ips for this host defined')) + + return result + + @require_context def fixed_ip_get_by_address(context, address, session=None): if not session: -- cgit From 5812a95736b9a16733b99700e8664dd29ae34def Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 18 Feb 2011 22:10:06 +0100 Subject: Introduce IptablesManager in linux_net. Port every use of iptables in linux_net to it. --- nova/network/linux_net.py | 287 ++++++++++++++++++++++++++++----------------- nova/tests/test_network.py | 59 ++++++++++ nova/utils.py | 61 ++++++---- nova/virt/libvirt_conn.py | 28 +++-- 4 files changed, 287 insertions(+), 148 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index c1cbff7d8..3d267d941 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -17,6 +17,7 @@ Implements vlans, bridges, and iptables rules using linux utilities. """ +import inspect import os from nova import db @@ -25,7 +26,6 @@ from nova import flags from nova import log as logging from nova import utils - LOG = logging.getLogger("nova.linux_net") @@ -63,73 +63,168 @@ flags.DEFINE_string('dmz_cidr', '10.128.0.0/24', 'dmz range that should be accepted') +binary_name = os.path.basename(inspect.stack()[-1][1]) + + +class IptablesRule(object): + def __init__(self, chain, rule, wrap=True): + self.chain = chain + self.rule = rule + self.wrap = wrap + + def __eq__(self, other): + return ((self.chain == other.chain) and + (self.rule == other.rule) and + (self.wrap == other.wrap)) + + def __ne__(self, other): + return ((self.chain != other.chain) or + (self.rule != other.rule) or + (self.wrap != other.wrap)) + + def __str__(self): + if self.wrap: + chain = '%s-%s' % (binary_name, self.chain) + else: + chain = self.chain + return '-A %s %s' % (chain, self.rule) + + +class IptablesTable(object): + def __init__(self): + self.rules = [] + self.chains = set() + + def add_chain(self, name): + self.chains.add(name) + + def remove_chain(self, name): + self.chains.remove(name) + + def add_rule(self, chain, rule, wrap=True): + if wrap and chain not in self.chains: + raise ValueError(_("Unknown chain: %r") % chain) + + self.rules.append(IptablesRule(chain, rule, wrap)) + + def remove_rule(self, chain, rule): + self.rules.remove(IptablesRule(chain, rule)) + +class IptablesManager(object): + def __init__(self, execute=None): + if not execute: + if FLAGS.fake_network: + self.execute = lambda *args, **kwargs: ('', '') + else: + self.execute = utils.execute + else: + self.execute = execute + + self.ipv4 = { 'filter': IptablesTable(), + 'nat': IptablesTable() } + self.ipv6 = { 'filter': IptablesTable(), + 'nat': IptablesTable() } + + self.ipv4['nat'].add_chain('SNATTING') + self.ipv4['nat'].add_rule('POSTROUTING', + '-j %s-SNATTING' % (binary_name,), + wrap=False) + + self.ipv4['filter'].add_chain('local') + self.ipv4['filter'].add_rule('FORWARD', + '-j %s-local' % (binary_name,), + wrap=False) + + # Wrap the builtin chains + builtin_chains = {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], + 'nat': ['PREROUTING', 'INPUT', + 'OUTPUT', 'POSTROUTING']} + + for table, chains in builtin_chains.iteritems(): + for chain in chains: + self.ipv4[table].add_chain(chain) + self.ipv4[table].add_rule(chain, + '-j %s-%s' % (binary_name, chain), + wrap=False) + + + def apply(self): + s = [('iptables', self.ipv4)] + if FLAGS.use_ipv6: + s += [('ip6tables', self.ipv6)] + + for cmd, tables in s: + for table in tables: + current_filter, _ = self.execute('sudo %s-save -t %s' % + (cmd, table), attempts=5) + current_lines = current_filter.split('\n') + new_filter = self.modify_rules(current_lines, tables[table]) + self.execute('sudo %s-restore' % (cmd,), + process_input='\n'.join(new_filter), + attempts=5) + + def modify_rules(self, current_lines, table, binary=None): + + chains = table.chains + rules = table.rules + + # Remove any trace of our rules + new_filter = filter(lambda l: '%s' % binary not in l, current_lines) + + seen_chains = False + for rules_index in range(len(new_filter)): + if not seen_chains: + if new_filter[rules_index].startswith(':'): + seen_chains = True + elif seen_chains == 1: + if not new_filter[rules_index].startswith(':'): + break + + new_filter[rules_index:rules_index] = [str(rule) for rule in rules] + new_filter[rules_index:rules_index] = [':%s-%s - [0:0]' % \ + (binary_name, name,) \ + for name in chains] + + return new_filter + + +iptables_manager = IptablesManager() + + def metadata_forward(): """Create forwarding rule for metadata""" - _confirm_rule("PREROUTING", "-t nat -s 0.0.0.0/0 " - "-d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT " - "--to-destination %s:%s" % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) + iptables_manager.ipv4['nat'].add_rule("PREROUTING", + "-s 0.0.0.0/0 -d 169.254.169.254/32 " + "-p tcp -m tcp --dport 80 -j DNAT " + "--to-destination %s:%s" % \ + (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) + iptables_manager.apply() def init_host(): """Basic networking setup goes here""" - if FLAGS.use_nova_chains: - _execute("sudo iptables -N nova_input", check_exit_code=False) - _execute("sudo iptables -D %s -j nova_input" % FLAGS.input_chain, - check_exit_code=False) - _execute("sudo iptables -A %s -j nova_input" % FLAGS.input_chain) - - _execute("sudo iptables -N nova_forward", check_exit_code=False) - _execute("sudo iptables -D FORWARD -j nova_forward", - check_exit_code=False) - _execute("sudo iptables -A FORWARD -j nova_forward") - - _execute("sudo iptables -N nova_output", check_exit_code=False) - _execute("sudo iptables -D OUTPUT -j nova_output", - check_exit_code=False) - _execute("sudo iptables -A OUTPUT -j nova_output") - - _execute("sudo iptables -t nat -N nova_prerouting", - check_exit_code=False) - _execute("sudo iptables -t nat -D PREROUTING -j nova_prerouting", - check_exit_code=False) - _execute("sudo iptables -t nat -A PREROUTING -j nova_prerouting") - - _execute("sudo iptables -t nat -N nova_postrouting", - check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j nova_postrouting", - check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j nova_postrouting") - - _execute("sudo iptables -t nat -N nova_snatting", - check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j nova_snatting", - check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j nova_snatting") - - _execute("sudo iptables -t nat -N nova_output", check_exit_code=False) - _execute("sudo iptables -t nat -D OUTPUT -j nova_output", - check_exit_code=False) - _execute("sudo iptables -t nat -A OUTPUT -j nova_output") - else: - # NOTE(vish): This makes it easy to ensure snatting rules always - # come after the accept rules in the postrouting chain - _execute("sudo iptables -t nat -N SNATTING", - check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j SNATTING", - check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j SNATTING") - # NOTE(devcamcar): Cloud public SNAT entries and the default # SNAT rule for outbound traffic. - _confirm_rule("SNATTING", "-t nat -s %s " - "-j SNAT --to-source %s" - % (FLAGS.fixed_range, FLAGS.routing_source_ip), append=True) + iptables_manager.ipv4['nat'].add_rule("SNATTING", + "-s %s -j SNAT --to-source %s" % \ + (FLAGS.fixed_range, + FLAGS.routing_source_ip)) + + iptables_manager.ipv4['nat'].add_rule("POSTROUTING", + "-s %s -j SNAT --to-source %s" % \ + (FLAGS.fixed_range, + FLAGS.routing_source_ip)) - _confirm_rule("POSTROUTING", "-t nat -s %s -d %s -j ACCEPT" % - (FLAGS.fixed_range, FLAGS.dmz_cidr)) - _confirm_rule("POSTROUTING", "-t nat -s %(range)s -d %(range)s -j ACCEPT" % - {'range': FLAGS.fixed_range}) + iptables_manager.ipv4['nat'].add_rule("POSTROUTING", + "-s %s -d %s -j ACCEPT" % \ + (FLAGS.fixed_range, FLAGS.dmz_cidr)) + + iptables_manager.ipv4['nat'].add_rule("POSTROUTING", + "-s %(range)s -d %(range)s " + "-j ACCEPT" % \ + {'range': FLAGS.fixed_range}) + iptables_manager.apply() def bind_floating_ip(floating_ip, check_exit_code=True): @@ -147,32 +242,33 @@ def unbind_floating_ip(floating_ip): def ensure_vlan_forward(public_ip, port, private_ip): """Sets up forwarding rules for vlan""" - _confirm_rule("FORWARD", "-d %s -p udp --dport 1194 -j ACCEPT" % - private_ip) - _confirm_rule("PREROUTING", - "-t nat -d %s -p udp --dport %s -j DNAT --to %s:1194" - % (public_ip, port, private_ip)) + iptables_manager.ipv4['filter'].add_rule("FORWARD", + "-d %s -p udp " + "--dport 1194 " + "-j ACCEPT" % private_ip) + iptables_manager.ipv4['nat'].add_rule("PREROUTING", + "-d %s -p udp " + "--dport %s -j DNAT --to %s:1194" % + (public_ip, port, private_ip)) + iptables_manager.apply() def ensure_floating_forward(floating_ip, fixed_ip): """Ensure floating ip forwarding rule""" - _confirm_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _confirm_rule("OUTPUT", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _confirm_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" - % (fixed_ip, floating_ip)) - + for chain, rule in floating_forward_rules(floating_ip, fixed_ip): + iptables_manager.ipv4['nat'].add_rule(chain, rule) + iptables_manager.apply() def remove_floating_forward(floating_ip, fixed_ip): """Remove forwarding for floating ip""" - _remove_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _remove_rule("OUTPUT", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _remove_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" - % (fixed_ip, floating_ip)) + for chain, rule in floating_forward_rules(floating_ip, fixed_ip): + iptables_manager.ipv4['nat'].remove_rule(chain, rule) + iptables_manager.apply() +def floating_forward_rules(floating_ip, fixed_ip): + return [("PREROUTING", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), + ("OUTPUT", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), + ("SNATTING", "-d %s -j DNAT --to %s" % (fixed_ip, floating_ip))] def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): """Create a vlan and bridge unless they already exist""" @@ -258,19 +354,12 @@ def ensure_bridge(bridge, interface, net_attrs=None): "enslave it to bridge %s.\n" % (interface, bridge)): raise exception.Error("Failed to add interface: %s" % err) - if FLAGS.use_nova_chains: - (out, err) = _execute("sudo iptables -N nova_forward", - check_exit_code=False) - if err != 'iptables: Chain already exists.\n': - # NOTE(vish): chain didn't exist link chain - _execute("sudo iptables -D FORWARD -j nova_forward", - check_exit_code=False) - _execute("sudo iptables -A FORWARD -j nova_forward") - - _confirm_rule("FORWARD", "--in-interface %s -j ACCEPT" % bridge) - _confirm_rule("FORWARD", "--out-interface %s -j ACCEPT" % bridge) - _execute("sudo iptables -N nova-local", check_exit_code=False) - _confirm_rule("FORWARD", "-j nova-local") + iptables_manager.ipv4['filter'].add_rule("FORWARD", + "--in-interface %s -j ACCEPT" % \ + bridge) + iptables_manager.ipv4['filter'].add_rule("FORWARD", + "--out-interface %s -j ACCEPT" % \ + bridge) def get_dhcp_hosts(context, network_id): @@ -390,26 +479,6 @@ def _device_exists(device): return not err -def _confirm_rule(chain, cmd, append=False): - """Delete and re-add iptables rule""" - if FLAGS.use_nova_chains: - chain = "nova_%s" % chain.lower() - if append: - loc = "-A" - else: - loc = "-I" - _execute("sudo iptables --delete %s %s" % (chain, cmd), - check_exit_code=False) - _execute("sudo iptables %s %s %s" % (loc, chain, cmd)) - - -def _remove_rule(chain, cmd): - """Remove iptables rule""" - if FLAGS.use_nova_chains: - chain = "%s" % chain.lower() - _execute("sudo iptables --delete %s %s" % (chain, cmd)) - - def _dnsmasq_cmd(net): """Builds dnsmasq command""" cmd = ['sudo -E dnsmasq', diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 00f9323f3..b28d64245 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -29,11 +29,70 @@ from nova import log as logging from nova import test from nova import utils from nova.auth import manager +from nova.network import linux_net FLAGS = flags.FLAGS LOG = logging.getLogger('nova.tests.network') +class IptablesManagerTestCase(test.TestCase): + sample_filter = """# Completed on Fri Feb 18 15:17:05 2011 +# Generated by iptables-save v1.4.10 on Fri Feb 18 15:17:05 2011 +*filter +:INPUT ACCEPT [2223527:305688874] +:FORWARD ACCEPT [0:0] +:OUTPUT ACCEPT [2172501:140856656] +-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT +-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT +-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT +-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT +-A FORWARD -d 192.168.122.0/24 -o virbr0 -m state --state RELATED,ESTABLISHED -j ACCEPT +-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT +-A FORWARD -i virbr0 -o virbr0 -j ACCEPT +-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable +-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable +COMMIT +# Completed on Fri Feb 18 15:17:05 2011""" + + def setUp(self): + super(IptablesManagerTestCase, self).setUp() + self.manager = linux_net.IptablesManager() + + def test_rules_are_wrapped(self): + current_lines = self.sample_filter.split('\n') + + table = self.manager.ipv4['filter'] + table.add_rule('FORWARD', '-s 1.2.3.4/5 -j DROP') + new_lines = self.manager.modify_rules(current_lines, table) + self.assertTrue('-A run_tests.py-FORWARD ' + '-s 1.2.3.4/5 -j DROP' in new_lines) + + table.remove_rule('FORWARD', '-s 1.2.3.4/5 -j DROP') + new_lines = self.manager.modify_rules(current_lines, table) + self.assertTrue('-A run_tests.py-FORWARD ' + '-s 1.2.3.4/5 -j DROP' not in new_lines) + + def test_wrapper_rules_in_place(self): + current_lines = self.sample_filter.split('\n') + + # TODO(soren): Add stuff for ipv6 + check_matrix = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], + 'nat': ['PREROUTING', 'INPUT', + 'OUTPUT', 'POSTROUTING']} } + + for ip_version in check_matrix: + ip = getattr(self.manager, 'ipv%d' % ip_version) + for table_name in ip: + table = ip[table_name] + new_lines = self.manager.modify_rules(current_lines, table) + for chain in check_matrix[ip_version][table_name]: + self.assertTrue(':run_tests.py-%s - [0:0]' % \ + (chain,) in new_lines) + self.assertTrue('-A %s -j run_tests.py-%s' % \ + (chain, chain) in new_lines) + print '\n'.join(new_lines) + + class NetworkTestCase(test.TestCase): """Test cases for network code""" def setUp(self): diff --git a/nova/utils.py b/nova/utils.py index ba71ebf39..bf3a4b098 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -124,32 +124,41 @@ def fetchfile(url, target): execute("curl --fail %s -o %s" % (url, target)) -def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (subprocess): %s"), cmd) - env = os.environ.copy() - if addl_env: - env.update(addl_env) - obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) - result = None - if process_input != None: - result = obj.communicate(process_input) - else: - result = obj.communicate() - obj.stdin.close() - if obj.returncode: - LOG.debug(_("Result was %s") % obj.returncode) - if check_exit_code and obj.returncode != 0: - (stdout, stderr) = result - raise ProcessExecutionError(exit_code=obj.returncode, - stdout=stdout, - stderr=stderr, - cmd=cmd) - # NOTE(termie): this appears to be necessary to let the subprocess call - # clean something up in between calls, without it two - # execute calls in a row hangs the second one - greenthread.sleep(0) - return result +def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, attempts=1): + while attempts > 0: + attempts -= 1 + try: + LOG.debug(_("Running cmd (subprocess): %s"), cmd) + env = os.environ.copy() + if addl_env: + env.update(addl_env) + obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + result = None + if process_input != None: + result = obj.communicate(process_input) + else: + result = obj.communicate() + obj.stdin.close() + if obj.returncode: + LOG.debug(_("Result was %s") % obj.returncode) + if check_exit_code and obj.returncode != 0: + (stdout, stderr) = result + raise ProcessExecutionError(exit_code=obj.returncode, + stdout=stdout, + stderr=stderr, + cmd=cmd) + # NOTE(termie): this appears to be necessary to let the subprocess call + # clean something up in between calls, without it two + # execute calls in a row hangs the second one + greenthread.sleep(0) + return result + except ProcessExecutionError: + if not attempts: + raise + else: + greenthread.sleep(random.randint(50,300)/100) + pass def ssh_execute(ssh, cmd, process_input=None, diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 7548fff63..11b3acbf6 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -64,7 +64,6 @@ from nova.compute import power_state from nova.virt import disk from nova.virt import images -libvirt_semaphore = semaphore.Semaphore() libvirt = None libxml2 = None Template = None @@ -1239,19 +1238,22 @@ class IptablesFirewallDriver(FirewallDriver): self.apply_ruleset() def apply_ruleset(self): - with libvirt_semaphore: - current_filter, _ = self.execute('sudo iptables-save -t filter') + current_filter, _ = self.execute('sudo iptables-save -t filter', + attempts=5) + current_lines = current_filter.split('\n') + new_filter = self.modify_rules(current_lines, 4) + self.execute('sudo iptables-restore', + process_input='\n'.join(new_filter), + attempts=5) + if(FLAGS.use_ipv6): + current_filter, _ = self.execute('sudo ip6tables-save ' + '-t filter', + attempts=5) current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines, 4) - self.execute('sudo iptables-restore', - process_input='\n'.join(new_filter)) - if(FLAGS.use_ipv6): - current_filter, _ = self.execute('sudo ip6tables-save ' - '-t filter') - current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines, 6) - self.execute('sudo ip6tables-restore', - process_input='\n'.join(new_filter)) + new_filter = self.modify_rules(current_lines, 6) + self.execute('sudo ip6tables-restore', + process_input='\n'.join(new_filter), + attempts=5) def modify_rules(self, current_lines, ip_version=4): ctxt = context.get_admin_context() -- cgit From cfd6d4e403dcb2405cd7ff48bad3083a02159d2c Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sat, 19 Feb 2011 00:14:08 +0100 Subject: Port libvirt_conn.IptablesDriver over to use linux_net.IptablesManager --- nova/network/linux_net.py | 17 +++- nova/tests/test_virt.py | 55 ++++++++---- nova/virt/libvirt_conn.py | 215 ++++++++++++++++++++-------------------------- 3 files changed, 145 insertions(+), 142 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 3d267d941..c11d34922 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -100,13 +100,23 @@ class IptablesTable(object): def remove_chain(self, name): self.chains.remove(name) + self.rules = filter(lambda r: r.chain != name, self.rules) def add_rule(self, chain, rule, wrap=True): if wrap and chain not in self.chains: raise ValueError(_("Unknown chain: %r") % chain) + if '$' in rule: + rule = ' '.join(map(self._wrap_target_chain, rule.split(' '))) + + print 'Adding rule: %r' % rule self.rules.append(IptablesRule(chain, rule, wrap)) + def _wrap_target_chain(self, s): + if s.startswith('$'): + return '%s-%s' % (binary_name, s[1:]) + return s + def remove_rule(self, chain, rule): self.rules.remove(IptablesRule(chain, rule)) @@ -122,8 +132,7 @@ class IptablesManager(object): self.ipv4 = { 'filter': IptablesTable(), 'nat': IptablesTable() } - self.ipv6 = { 'filter': IptablesTable(), - 'nat': IptablesTable() } + self.ipv6 = { 'filter': IptablesTable() } self.ipv4['nat'].add_chain('SNATTING') self.ipv4['nat'].add_rule('POSTROUTING', @@ -135,6 +144,10 @@ class IptablesManager(object): '-j %s-local' % (binary_name,), wrap=False) + self.ipv4['filter'].add_rule('OUTPUT', + '-j %s-local' % (binary_name,), + wrap=False) + # Wrap the builtin chains builtin_chains = {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], 'nat': ['PREROUTING', 'INPUT', diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 6e5a0114b..a88e01818 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import re from xml.etree.ElementTree import fromstring as xml_to_tree from xml.dom.minidom import parseString as xml_to_dom @@ -233,16 +234,22 @@ class IptablesFirewallTestCase(test.TestCase): self.manager.delete_user(self.user) super(IptablesFirewallTestCase, self).tearDown() - in_rules = [ + in_nat_rules = [ + '# Generated by iptables-save v1.4.10 on Sat Feb 19 00:03:19 2011', + '*nat', + ':PREROUTING ACCEPT [1170:189210]', + ':INPUT ACCEPT [844:71028]', + ':OUTPUT ACCEPT [5149:405186]', + ':POSTROUTING ACCEPT [5063:386098]' + ] + + in_filter_rules = [ '# Generated by iptables-save v1.4.4 on Mon Dec 6 11:54:13 2010', '*filter', ':INPUT ACCEPT [969615:281627771]', ':FORWARD ACCEPT [0:0]', ':OUTPUT ACCEPT [915599:63811649]', ':nova-block-ipv4 - [0:0]', - '-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT ', - '-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT ', - '-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT ', '-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT ', '-A FORWARD -d 192.168.122.0/24 -o virbr0 -m state --state RELATED' ',ESTABLISHED -j ACCEPT ', @@ -254,7 +261,7 @@ class IptablesFirewallTestCase(test.TestCase): '# Completed on Mon Dec 6 11:54:13 2010', ] - in6_rules = [ + in6_filter_rules = [ '# Generated by ip6tables-save v1.4.4 on Tue Jan 18 23:47:56 2011', '*filter', ':INPUT ACCEPT [349155:75810423]', @@ -314,23 +321,31 @@ class IptablesFirewallTestCase(test.TestCase): instance_ref = db.instance_get(admin_ctxt, instance_ref['id']) # self.fw.add_instance(instance_ref) - def fake_iptables_execute(cmd, process_input=None): + def fake_iptables_execute(cmd, process_input=None, attempts=5): if cmd == 'sudo ip6tables-save -t filter': - return '\n'.join(self.in6_rules), None + return '\n'.join(self.in6_filter_rules), None if cmd == 'sudo iptables-save -t filter': - return '\n'.join(self.in_rules), None + return '\n'.join(self.in_filter_rules), None + if cmd == 'sudo iptables-save -t nat': + return '\n'.join(self.in_nat_rules), None if cmd == 'sudo iptables-restore': - self.out_rules = process_input.split('\n') + lines = process_input.split('\n') + if '*filter' in lines: + self.out_rules = lines return '', '' if cmd == 'sudo ip6tables-restore': - self.out6_rules = process_input.split('\n') + lines = process_input.split('\n') + if '*filter' in lines: + self.out6_rules = lines return '', '' - self.fw.execute = fake_iptables_execute + + from nova.network import linux_net + linux_net.iptables_manager.execute = fake_iptables_execute self.fw.prepare_instance_filter(instance_ref) self.fw.apply_instance_filter(instance_ref) - in_rules = filter(lambda l: not l.startswith('#'), self.in_rules) + in_rules = filter(lambda l: not l.startswith('#'), self.in_filter_rules) for rule in in_rules: if not 'nova' in rule: self.assertTrue(rule in self.out_rules, @@ -338,6 +353,7 @@ class IptablesFirewallTestCase(test.TestCase): instance_chain = None for rule in self.out_rules: + print rule # This is pretty crude, but it'll do for now if '-d 10.11.12.13 -j' in rule: instance_chain = rule.split(' ')[-1] @@ -353,17 +369,18 @@ class IptablesFirewallTestCase(test.TestCase): self.assertTrue(security_group_chain, "The security group chain wasn't added") - self.assertTrue('-A %s -p icmp -s 192.168.11.0/24 -j ACCEPT' % \ - security_group_chain in self.out_rules, + regex = re.compile('-A .* -p icmp -s 192.168.11.0/24 -j ACCEPT') + self.assertTrue(len(filter(regex.match, self.out_rules)) > 0, "ICMP acceptance rule wasn't added") - self.assertTrue('-A %s -p icmp -s 192.168.11.0/24 -m icmp --icmp-type ' - '8 -j ACCEPT' % security_group_chain in self.out_rules, + regex = re.compile('-A .* -p icmp -s 192.168.11.0/24 -m icmp ' + '--icmp-type 8 -j ACCEPT') + self.assertTrue(len(filter(regex.match, self.out_rules)) > 0, "ICMP Echo Request acceptance rule wasn't added") - self.assertTrue('-A %s -p tcp -s 192.168.10.0/24 -m multiport ' - '--dports 80:81 -j ACCEPT' % security_group_chain \ - in self.out_rules, + regex = re.compile('-A .* -p tcp -s 192.168.10.0/24 -m multiport ' + '--dports 80:81 -j ACCEPT') + self.assertTrue(len(filter(regex.match, self.out_rules)) > 0, "TCP port 80/81 acceptance rule wasn't added") diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 11b3acbf6..976ccaca5 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -57,7 +57,6 @@ from nova import exception from nova import flags from nova import log as logging from nova import utils -#from nova.api import context from nova.auth import manager from nova.compute import instance_types from nova.compute import power_state @@ -1207,10 +1206,14 @@ class NWFilterFirewall(FirewallDriver): class IptablesFirewallDriver(FirewallDriver): def __init__(self, execute=None, **kwargs): - self.execute = execute or utils.execute + from nova.network import linux_net + self.iptables = linux_net.iptables_manager self.instances = {} self.nwfilter = NWFilterFirewall(kwargs['get_connection']) + self.iptables.ipv4['filter'].add_chain('sg-fallback') + self.iptables.ipv4['filter'].add_rule('sg-fallback', '-j DROP') + def setup_basic_filtering(self, instance): """Use NWFilter from libvirt for this.""" return self.nwfilter.setup_basic_filtering(instance) @@ -1226,124 +1229,92 @@ class IptablesFirewallDriver(FirewallDriver): LOG.info(_('Attempted to unfilter instance %s which is not ' 'filtered'), instance['id']) - def add_instance(self, instance): + def prepare_instance_filter(self, instance): self.instances[instance['id']] = instance + chain_name = self._instance_chain_name(instance) + + self.iptables.ipv4['filter'].add_chain(chain_name) + ipv4_address = self._ip_for_instance(instance) + self.iptables.ipv4['filter'].add_rule('local', + '-d %s -j $%s' % + (ipv4_address, chain_name)) + + if FLAGS.use_ipv6: + self.iptables.ipv6['filter'].add_chain(chain_name) + ipv6_address = self._ip_for_instance_v6(instance) + self.iptables.ipv4['filter'].add_rule('local', + '-d %s -j $%s' % + (ipv6_address, + chain_name)) + + ipv4_rules, ipv6_rules = self.instance_rules(instance) + + for rule in ipv4_rules: + self.iptables.ipv4['filter'].add_rule(chain_name, rule) + + if FLAGS.use_ipv6: + for rule in ipv6_rules: + self.iptables.ipv6['filter'].add_rule(chain_name, rule) + + self.iptables.apply() + def unfilter_instance(self, instance): - self.remove_instance(instance) - self.apply_ruleset() + chain_name = self._instance_chain_name(instance) - def prepare_instance_filter(self, instance): - self.add_instance(instance) - self.apply_ruleset() + self.iptables.ipv4['filter'].remove_chain(chain_name) + if FLAGS.use_ipv6: + self.iptables.ipv6['filter'].remove_chain(chain_name) - def apply_ruleset(self): - current_filter, _ = self.execute('sudo iptables-save -t filter', - attempts=5) - current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines, 4) - self.execute('sudo iptables-restore', - process_input='\n'.join(new_filter), - attempts=5) - if(FLAGS.use_ipv6): - current_filter, _ = self.execute('sudo ip6tables-save ' - '-t filter', - attempts=5) - current_lines = current_filter.split('\n') - new_filter = self.modify_rules(current_lines, 6) - self.execute('sudo ip6tables-restore', - process_input='\n'.join(new_filter), - attempts=5) - - def modify_rules(self, current_lines, ip_version=4): + self.iptables.apply() + + + def instance_rules(self, instance): ctxt = context.get_admin_context() - # Remove any trace of nova rules. - new_filter = filter(lambda l: 'nova-' not in l, current_lines) - - seen_chains = False - for rules_index in range(len(new_filter)): - if not seen_chains: - if new_filter[rules_index].startswith(':'): - seen_chains = True - elif seen_chains == 1: - if not new_filter[rules_index].startswith(':'): - break - our_chains = [':nova-fallback - [0:0]'] - our_rules = ['-A nova-fallback -j DROP'] - - our_chains += [':nova-local - [0:0]'] - our_rules += ['-A FORWARD -j nova-local'] - our_rules += ['-A OUTPUT -j nova-local'] - - security_groups = {} - # Add our chains - # First, we add instance chains and rules - for instance_id in self.instances: - instance = self.instances[instance_id] - chain_name = self._instance_chain_name(instance) - if(ip_version == 4): - ip_address = self._ip_for_instance(instance) - elif(ip_version == 6): - ip_address = self._ip_for_instance_v6(instance) - - our_chains += [':%s - [0:0]' % chain_name] - - # Jump to the per-instance chain - our_rules += ['-A nova-local -d %s -j %s' % (ip_address, - chain_name)] - - # Always drop invalid packets - our_rules += ['-A %s -m state --state ' - 'INVALID -j DROP' % (chain_name,)] - - # Allow established connections - our_rules += ['-A %s -m state --state ' - 'ESTABLISHED,RELATED -j ACCEPT' % (chain_name,)] - - # Jump to each security group chain in turn - for security_group in \ - db.security_group_get_by_instance(ctxt, - instance['id']): - security_groups[security_group['id']] = security_group - - sg_chain_name = self._security_group_chain_name( - security_group['id']) + ipv4_rules = [] + ipv6_rules = [] - our_rules += ['-A %s -j %s' % (chain_name, sg_chain_name)] - - if(ip_version == 4): - # Allow DHCP responses - dhcp_server = self._dhcp_server_for_instance(instance) - our_rules += ['-A %s -s %s -p udp --sport 67 --dport 68 ' - '-j ACCEPT ' % (chain_name, dhcp_server)] - #Allow project network traffic - if (FLAGS.allow_project_net_traffic): - cidr = self._project_cidr_for_instance(instance) - our_rules += ['-A %s -s %s -j ACCEPT' % (chain_name, cidr)] - elif(ip_version == 6): - # Allow RA responses - ra_server = self._ra_server_for_instance(instance) - if ra_server: - our_rules += ['-A %s -s %s -p icmpv6 -j ACCEPT' % - (chain_name, ra_server + "/128")] - #Allow project network traffic - if (FLAGS.allow_project_net_traffic): - cidrv6 = self._project_cidrv6_for_instance(instance) - our_rules += ['-A %s -s %s -j ACCEPT' % - (chain_name, cidrv6)] - - # If nothing matches, jump to the fallback chain - our_rules += ['-A %s -j nova-fallback' % (chain_name,)] + # Always drop invalid packets + ipv4_rules += ['-m state --state ' 'INVALID -j DROP'] + ipv6_rules += ['-m state --state ' 'INVALID -j DROP'] - # then, security group chains and rules - for security_group_id in security_groups: - chain_name = self._security_group_chain_name(security_group_id) - our_chains += [':%s - [0:0]' % chain_name] + # Allow established connections + ipv4_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT'] + ipv6_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT'] + + dhcp_server = self._dhcp_server_for_instance(instance) + ipv4_rules += ['-s %s -p udp --sport 67 --dport 68 ' + '-j ACCEPT' % (dhcp_server,)] + + #Allow project network traffic + if FLAGS.allow_project_net_traffic: + cidr = self._project_cidr_for_instance(instance) + ipv4_rules += ['-s %s -j ACCEPT' % (cidr,)] + + # We wrap these in FLAGS.use_ipv6 because they might cause + # a DB lookup. The other ones are just list operations, so + # they're not worth the clutter. + if FLAGS.use_ipv6: + # Allow RA responses + ra_server = self._ra_server_for_instance(instance) + if ra_server: + ipv6_rules += ['-s %s/128 -p icmpv6 -j ACCEPT' % (ra_server,)] - rules = \ - db.security_group_rule_get_by_security_group(ctxt, - security_group_id) + #Allow project network traffic + if FLAGS.allow_project_net_traffic: + cidrv6 = self._project_cidrv6_for_instance(instance) + ipv6_rules += ['-s %s -j ACCEPT' % (cidrv6,)] + + + security_groups = db.security_group_get_by_instance(ctxt, + instance['id']) + + + # then, security group chains and rules + for security_group in security_groups: + rules = db.security_group_rule_get_by_security_group(ctxt, + security_group['id']) for rule in rules: logging.info('%r', rule) @@ -1354,14 +1325,16 @@ class IptablesFirewallDriver(FirewallDriver): continue version = _get_ip_version(rule.cidr) - if version != ip_version: - continue + if version == 4: + rules = ipv4_rules + else: + rules = ipv6_rules protocol = rule.protocol if version == 6 and rule.protocol == 'icmp': protocol = 'icmpv6' - args = ['-A', chain_name, '-p', protocol, '-s', rule.cidr] + args = ['-p', protocol, '-s', rule.cidr] if rule.protocol in ['udp', 'tcp']: if rule.from_port == rule.to_port: @@ -1382,20 +1355,20 @@ class IptablesFirewallDriver(FirewallDriver): icmp_type_arg += '/%s' % icmp_code if icmp_type_arg: - if(ip_version == 4): + if version == 4: args += ['-m', 'icmp', '--icmp-type', icmp_type_arg] - elif(ip_version == 6): + elif version == 6: args += ['-m', 'icmp6', '--icmpv6-type', icmp_type_arg] args += ['-j ACCEPT'] - our_rules += [' '.join(args)] + rules += [' '.join(args)] + + ipv4_rules += ['-j $fallback'] + ipv6_rules += ['-j $fallback'] - new_filter[rules_index:rules_index] = our_rules - new_filter[rules_index:rules_index] = our_chains - logging.info('new_filter: %s', '\n'.join(new_filter)) - return new_filter + return ipv4_rules, ipv6_rules def refresh_security_group_members(self, security_group): pass @@ -1407,7 +1380,7 @@ class IptablesFirewallDriver(FirewallDriver): return 'nova-sg-%s' % (security_group_id,) def _instance_chain_name(self, instance): - return 'nova-inst-%s' % (instance['id'],) + return 'inst-%s' % (instance['id'],) def _ip_for_instance(self, instance): return db.instance_get_fixed_address(context.get_admin_context(), -- cgit From 99760bd7a51371b29cf0f76134187dc81e7545d0 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sat, 19 Feb 2011 00:30:44 +0100 Subject: Rename a few things for more clarity. --- nova/network/linux_net.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index c11d34922..b657ab4bc 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -109,7 +109,6 @@ class IptablesTable(object): if '$' in rule: rule = ' '.join(map(self._wrap_target_chain, rule.split(' '))) - print 'Adding rule: %r' % rule self.rules.append(IptablesRule(chain, rule, wrap)) def _wrap_target_chain(self, s): @@ -168,9 +167,9 @@ class IptablesManager(object): for cmd, tables in s: for table in tables: - current_filter, _ = self.execute('sudo %s-save -t %s' % - (cmd, table), attempts=5) - current_lines = current_filter.split('\n') + current_table, _ = self.execute('sudo %s-save -t %s' % + (cmd, table), attempts=5) + current_lines = current_table.split('\n') new_filter = self.modify_rules(current_lines, tables[table]) self.execute('sudo %s-restore' % (cmd,), process_input='\n'.join(new_filter), @@ -182,7 +181,7 @@ class IptablesManager(object): rules = table.rules # Remove any trace of our rules - new_filter = filter(lambda l: '%s' % binary not in l, current_lines) + new_filter = filter(lambda l: binary_name not in l, current_lines) seen_chains = False for rules_index in range(len(new_filter)): -- cgit From 23729c543350ce4ce563077522f18d0bedd1e61b Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sat, 19 Feb 2011 00:36:34 +0100 Subject: Security group fallback is named sg-fallback. --- nova/virt/libvirt_conn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 976ccaca5..0ddf889a1 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1365,8 +1365,8 @@ class IptablesFirewallDriver(FirewallDriver): args += ['-j ACCEPT'] rules += [' '.join(args)] - ipv4_rules += ['-j $fallback'] - ipv6_rules += ['-j $fallback'] + ipv4_rules += ['-j $sg-fallback'] + ipv6_rules += ['-j $sg-fallback'] return ipv4_rules, ipv6_rules -- cgit From d0733621758985bdd621a05c7c8a53fe27aa62f2 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Sat, 19 Feb 2011 01:28:26 +0100 Subject: Wrap iptables calls in a semaphore. --- nova/network/linux_net.py | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index b81a704bf..ecda450bf 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -20,6 +20,8 @@ Implements vlans, bridges, and iptables rules using linux utilities. import inspect import os +from eventlet import semaphore + from nova import db from nova import exception from nova import flags @@ -149,8 +151,7 @@ class IptablesManager(object): # Wrap the builtin chains builtin_chains = {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], - 'nat': ['PREROUTING', 'INPUT', - 'OUTPUT', 'POSTROUTING']} + 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']} for table, chains in builtin_chains.iteritems(): for chain in chains: @@ -158,22 +159,24 @@ class IptablesManager(object): self.ipv4[table].add_rule(chain, '-j %s-%s' % (binary_name, chain), wrap=False) + self.semaphore = semaphore.Semaphore() def apply(self): - s = [('iptables', self.ipv4)] - if FLAGS.use_ipv6: - s += [('ip6tables', self.ipv6)] - - for cmd, tables in s: - for table in tables: - current_table, _ = self.execute('sudo %s-save -t %s' % - (cmd, table), attempts=5) - current_lines = current_table.split('\n') - new_filter = self.modify_rules(current_lines, tables[table]) - self.execute('sudo %s-restore' % (cmd,), - process_input='\n'.join(new_filter), - attempts=5) + with self.semaphore: + s = [('iptables', self.ipv4)] + if FLAGS.use_ipv6: + s += [('ip6tables', self.ipv6)] + + for cmd, tables in s: + for table in tables: + current_table, _ = self.execute('sudo %s-save -t %s' % + (cmd, table), attempts=5) + current_lines = current_table.split('\n') + new_filter = self.modify_rules(current_lines, tables[table]) + self.execute('sudo %s-restore' % (cmd,), + process_input='\n'.join(new_filter), + attempts=5) def modify_rules(self, current_lines, table, binary=None): -- cgit From e729c49543c5acf354b154a3e2d9fd76a2f7da35 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 09:17:33 +0100 Subject: Fix refresh sec groups. --- nova/virt/libvirt_conn.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 0ddf889a1..3faf01f4b 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1231,7 +1231,10 @@ class IptablesFirewallDriver(FirewallDriver): def prepare_instance_filter(self, instance): self.instances[instance['id']] = instance + self.add_filters_for_instance(instance) + self.iptables.apply() + def add_filters_for_instance(self, instance): chain_name = self._instance_chain_name(instance) self.iptables.ipv4['filter'].add_chain(chain_name) @@ -1257,18 +1260,17 @@ class IptablesFirewallDriver(FirewallDriver): for rule in ipv6_rules: self.iptables.ipv6['filter'].add_rule(chain_name, rule) + def unfilter_instance(self, instance): + self.remove_filters_for_instance(instance) self.iptables.apply() - def unfilter_instance(self, instance): + def remove_filters_for_instance(self, instance): chain_name = self._instance_chain_name(instance) self.iptables.ipv4['filter'].remove_chain(chain_name) if FLAGS.use_ipv6: self.iptables.ipv6['filter'].remove_chain(chain_name) - self.iptables.apply() - - def instance_rules(self, instance): ctxt = context.get_admin_context() @@ -1374,7 +1376,10 @@ class IptablesFirewallDriver(FirewallDriver): pass def refresh_security_group_rules(self, security_group): - self.apply_ruleset() + for instance in self.instances: + self.remove_filters_for_instance(instance) + self.add_filters_for_instance(instance) + self.iptables.apply() def _security_group_chain_name(self, security_group_id): return 'nova-sg-%s' % (security_group_id,) -- cgit From cbb0402efac4ededdda0ac2097ec087216e23931 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 10:18:43 +0100 Subject: Also remove rules that jump to deleted chains. --- nova/network/linux_net.py | 5 ++++- nova/virt/libvirt_conn.py | 7 ++----- 2 files changed, 6 insertions(+), 6 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index ecda450bf..1f96a4d55 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -104,6 +104,9 @@ class IptablesTable(object): self.chains.remove(name) self.rules = filter(lambda r: r.chain != name, self.rules) + jump_snippet = '-j %s-%s' % (binary_name, name) + self.rules = filter(lambda r: jump_snippet not in r.rule, self.rules) + def add_rule(self, chain, rule, wrap=True): if wrap and chain not in self.chains: raise ValueError(_("Unknown chain: %r") % chain) @@ -283,7 +286,7 @@ def remove_floating_forward(floating_ip, fixed_ip): def floating_forward_rules(floating_ip, fixed_ip): return [("PREROUTING", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), ("OUTPUT", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), - ("SNATTING", "-d %s -j DNAT --to %s" % (fixed_ip, floating_ip))] + ("SNATTING", "-d %s -j SNAT --to %s" % (fixed_ip, floating_ip))] def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): """Create a vlan and bridge unless they already exist""" diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 3faf01f4b..daf8f0ed7 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -44,9 +44,6 @@ import uuid from xml.dom import minidom -from eventlet import greenthread -from eventlet import event -from eventlet import semaphore from eventlet import tpool import IPy @@ -1246,7 +1243,7 @@ class IptablesFirewallDriver(FirewallDriver): if FLAGS.use_ipv6: self.iptables.ipv6['filter'].add_chain(chain_name) ipv6_address = self._ip_for_instance_v6(instance) - self.iptables.ipv4['filter'].add_rule('local', + self.iptables.ipv6['filter'].add_rule('local', '-d %s -j $%s' % (ipv6_address, chain_name)) @@ -1376,7 +1373,7 @@ class IptablesFirewallDriver(FirewallDriver): pass def refresh_security_group_rules(self, security_group): - for instance in self.instances: + for instance in self.instances.values(): self.remove_filters_for_instance(instance) self.add_filters_for_instance(instance) self.iptables.apply() -- cgit From 9eebe4317f86ae13ffeaca1622e9fc555bc28ebc Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 10:42:59 +0100 Subject: Unfilter instance correctly on termination. --- nova/network/linux_net.py | 4 ++++ nova/virt/libvirt_conn.py | 8 +++----- 2 files changed, 7 insertions(+), 5 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 1f96a4d55..1145bfa7a 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -101,6 +101,10 @@ class IptablesTable(object): self.chains.add(name) def remove_chain(self, name): + if name not in self.chain: + LOG.debug(_("Attempted to remove chain %s which doesn't exist"), + name) + return self.chains.remove(name) self.rules = filter(lambda r: r.chain != name, self.rules) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index daf8f0ed7..0c355e48e 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1219,9 +1219,11 @@ class IptablesFirewallDriver(FirewallDriver): """No-op. Everything is done in prepare_instance_filter""" pass - def remove_instance(self, instance): + def unfilter_instance(self, instance): if instance['id'] in self.instances: del self.instances[instance['id']] + self.remove_filters_for_instance(instance) + self.iptables.apply() else: LOG.info(_('Attempted to unfilter instance %s which is not ' 'filtered'), instance['id']) @@ -1257,10 +1259,6 @@ class IptablesFirewallDriver(FirewallDriver): for rule in ipv6_rules: self.iptables.ipv6['filter'].add_rule(chain_name, rule) - def unfilter_instance(self, instance): - self.remove_filters_for_instance(instance) - self.iptables.apply() - def remove_filters_for_instance(self, instance): chain_name = self._instance_chain_name(instance) -- cgit From 384a4525e9d6de54158cd170487fce95df814b52 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 11:01:27 +0100 Subject: Fix typo --- nova/network/linux_net.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 1145bfa7a..c5779f85f 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -101,7 +101,7 @@ class IptablesTable(object): self.chains.add(name) def remove_chain(self, name): - if name not in self.chain: + if name not in self.chains: LOG.debug(_("Attempted to remove chain %s which doesn't exist"), name) return -- cgit From 15203c9ceaa94f0cd5bad96622ee93af7662bcce Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 12:22:29 +0100 Subject: Allow non-existing rules to be removed. --- nova/network/linux_net.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index c5779f85f..d4cfbbde9 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -125,8 +125,12 @@ class IptablesTable(object): return '%s-%s' % (binary_name, s[1:]) return s - def remove_rule(self, chain, rule): - self.rules.remove(IptablesRule(chain, rule)) + def remove_rule(self, *args, **kwargs): + try: + self.rules.remove(IptablesRule(*args, **kwargs)) + except ValueError: + LOG.debug(_("Tried to remove rule that wasn't there: %r %r"), + args, kwargs) class IptablesManager(object): def __init__(self, execute=None): -- cgit From a57dffb5fdfbfac59b9ddbe7b33d6f03b7b748ba Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 14:16:42 +0100 Subject: PEP-8 fixes --- nova/network/linux_net.py | 28 ++++++++++++++++++++-------- nova/tests/test_network.py | 21 +++++++++------------ nova/tests/test_virt.py | 3 ++- nova/utils.py | 12 ++++++------ nova/virt/libvirt_conn.py | 2 -- 5 files changed, 37 insertions(+), 29 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index d4cfbbde9..b5d1323a1 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -129,8 +129,10 @@ class IptablesTable(object): try: self.rules.remove(IptablesRule(*args, **kwargs)) except ValueError: - LOG.debug(_("Tried to remove rule that wasn't there: %r %r"), - args, kwargs) + LOG.debug(_("Tried to remove rule that wasn't there:" + " %(args)r %(kwargs)r"), {'args': args, + 'kwargs': kwargs}) + class IptablesManager(object): def __init__(self, execute=None): @@ -142,9 +144,9 @@ class IptablesManager(object): else: self.execute = execute - self.ipv4 = { 'filter': IptablesTable(), - 'nat': IptablesTable() } - self.ipv6 = { 'filter': IptablesTable() } + self.ipv4 = {'filter': IptablesTable(), + 'nat': IptablesTable()} + self.ipv6 = {'filter': IptablesTable()} self.ipv4['nat'].add_chain('SNATTING') self.ipv4['nat'].add_rule('POSTROUTING', @@ -155,11 +157,18 @@ class IptablesManager(object): self.ipv4['filter'].add_rule('FORWARD', '-j %s-local' % (binary_name,), wrap=False) - self.ipv4['filter'].add_rule('OUTPUT', '-j %s-local' % (binary_name,), wrap=False) + self.ipv6['filter'].add_chain('local') + self.ipv6['filter'].add_rule('FORWARD', + '-j %s-local' % (binary_name,), + wrap=False) + self.ipv6['filter'].add_rule('OUTPUT', + '-j %s-local' % (binary_name,), + wrap=False) + # Wrap the builtin chains builtin_chains = {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']} @@ -172,7 +181,6 @@ class IptablesManager(object): wrap=False) self.semaphore = semaphore.Semaphore() - def apply(self): with self.semaphore: s = [('iptables', self.ipv4)] @@ -184,7 +192,8 @@ class IptablesManager(object): current_table, _ = self.execute('sudo %s-save -t %s' % (cmd, table), attempts=5) current_lines = current_table.split('\n') - new_filter = self.modify_rules(current_lines, tables[table]) + new_filter = self.modify_rules(current_lines, + tables[table]) self.execute('sudo %s-restore' % (cmd,), process_input='\n'.join(new_filter), attempts=5) @@ -285,17 +294,20 @@ def ensure_floating_forward(floating_ip, fixed_ip): iptables_manager.ipv4['nat'].add_rule(chain, rule) iptables_manager.apply() + def remove_floating_forward(floating_ip, fixed_ip): """Remove forwarding for floating ip""" for chain, rule in floating_forward_rules(floating_ip, fixed_ip): iptables_manager.ipv4['nat'].remove_rule(chain, rule) iptables_manager.apply() + def floating_forward_rules(floating_ip, fixed_ip): return [("PREROUTING", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), ("OUTPUT", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), ("SNATTING", "-d %s -j SNAT --to %s" % (fixed_ip, floating_ip))] + def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): """Create a vlan and bridge unless they already exist""" interface = ensure_vlan(vlan_num) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index b28d64245..c9a62a391 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -42,15 +42,14 @@ class IptablesManagerTestCase(test.TestCase): :INPUT ACCEPT [2223527:305688874] :FORWARD ACCEPT [0:0] :OUTPUT ACCEPT [2172501:140856656] --A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT --A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT --A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT --A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT --A FORWARD -d 192.168.122.0/24 -o virbr0 -m state --state RELATED,ESTABLISHED -j ACCEPT --A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT --A FORWARD -i virbr0 -o virbr0 -j ACCEPT --A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable --A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable +-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT +-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT +-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT +-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT +-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT +-A FORWARD -i virbr0 -o virbr0 -j ACCEPT +-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable +-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable COMMIT # Completed on Fri Feb 18 15:17:05 2011""" @@ -77,8 +76,7 @@ COMMIT # TODO(soren): Add stuff for ipv6 check_matrix = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], - 'nat': ['PREROUTING', 'INPUT', - 'OUTPUT', 'POSTROUTING']} } + 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}} for ip_version in check_matrix: ip = getattr(self.manager, 'ipv%d' % ip_version) @@ -90,7 +88,6 @@ COMMIT (chain,) in new_lines) self.assertTrue('-A %s -j run_tests.py-%s' % \ (chain, chain) in new_lines) - print '\n'.join(new_lines) class NetworkTestCase(test.TestCase): diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index a88e01818..11201788c 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -345,7 +345,8 @@ class IptablesFirewallTestCase(test.TestCase): self.fw.prepare_instance_filter(instance_ref) self.fw.apply_instance_filter(instance_ref) - in_rules = filter(lambda l: not l.startswith('#'), self.in_filter_rules) + in_rules = filter(lambda l: not l.startswith('#'), + self.in_filter_rules) for rule in in_rules: if not 'nova' in rule: self.assertTrue(rule in self.out_rules, diff --git a/nova/utils.py b/nova/utils.py index 644bf18fd..5b44bccb5 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -126,7 +126,8 @@ def fetchfile(url, target): execute("curl --fail %s -o %s" % (url, target)) -def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, attempts=1): +def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, + attempts=1): while attempts > 0: attempts -= 1 try: @@ -150,17 +151,16 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, attemp stdout=stdout, stderr=stderr, cmd=cmd) - # NOTE(termie): this appears to be necessary to let the subprocess call - # clean something up in between calls, without it two - # execute calls in a row hangs the second one + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one greenthread.sleep(0) return result except ProcessExecutionError: if not attempts: raise else: - greenthread.sleep(random.randint(50,300)/100) - pass + greenthread.sleep(random.randint(20, 200) / 100.0) def ssh_execute(ssh, cmd, process_input=None, diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 0c355e48e..7f74e3505 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1303,11 +1303,9 @@ class IptablesFirewallDriver(FirewallDriver): cidrv6 = self._project_cidrv6_for_instance(instance) ipv6_rules += ['-s %s -j ACCEPT' % (cidrv6,)] - security_groups = db.security_group_get_by_instance(ctxt, instance['id']) - # then, security group chains and rules for security_group in security_groups: rules = db.security_group_rule_get_by_security_group(ctxt, -- cgit From 3d2ec0f594e02018a32c8d0d7a8cc46f7ab4c849 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 14:39:02 +0100 Subject: Wrap ipv6 rules, too --- nova/network/linux_net.py | 26 +++++++++++++++++--------- nova/tests/test_network.py | 3 ++- 2 files changed, 19 insertions(+), 10 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index b5d1323a1..f47219b2e 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -170,15 +170,23 @@ class IptablesManager(object): wrap=False) # Wrap the builtin chains - builtin_chains = {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], - 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']} - - for table, chains in builtin_chains.iteritems(): - for chain in chains: - self.ipv4[table].add_chain(chain) - self.ipv4[table].add_rule(chain, - '-j %s-%s' % (binary_name, chain), - wrap=False) + builtin_chains = { 4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], + 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}, + 6: {'filter': ['INPUT', 'OUTPUT', 'FORWARD']}} + + for ip_version in builtin_chains: + if ip_version == 4: + tables = self.ipv4 + elif ip_version == 6: + tables = self.ipv6 + + for table, chains in builtin_chains[ip_version].iteritems(): + for chain in chains: + tables[table].add_chain(chain) + tables[table].add_rule(chain, + '-j %s-%s' % (binary_name, chain), + wrap=False) + self.semaphore = semaphore.Semaphore() def apply(self): diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index c9a62a391..f1d4fe133 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -76,7 +76,8 @@ COMMIT # TODO(soren): Add stuff for ipv6 check_matrix = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], - 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}} + 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}, + 6: {'filter': ['INPUT', 'OUTPUT', 'FORWARD']}} for ip_version in check_matrix: ip = getattr(self.manager, 'ipv%d' % ip_version) -- cgit From 1ed4df93a246a06518f2c216cd0fc60ca67eb5c4 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 14:39:38 +0100 Subject: More PEP-8 --- nova/network/linux_net.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index f47219b2e..d7a3075cb 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -170,9 +170,9 @@ class IptablesManager(object): wrap=False) # Wrap the builtin chains - builtin_chains = { 4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], - 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}, - 6: {'filter': ['INPUT', 'OUTPUT', 'FORWARD']}} + builtin_chains = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], + 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}, + 6: {'filter': ['INPUT', 'OUTPUT', 'FORWARD']}} for ip_version in builtin_chains: if ip_version == 4: -- cgit From fc0ea52d9379649d28de88d4fa1628e455533842 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 23:08:26 +0100 Subject: Add a bunch of docs for the new iptables hotness. --- nova/network/linux_net.py | 72 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 4 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index d7a3075cb..3b6ec9338 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -69,6 +69,11 @@ binary_name = os.path.basename(inspect.stack()[-1][1]) class IptablesRule(object): + """An iptables rule + + You shouldn't need to use this class directly, it's only used by + IptablesManager + """ def __init__(self, chain, rule, wrap=True): self.chain = chain self.rule = rule @@ -93,14 +98,33 @@ class IptablesRule(object): class IptablesTable(object): + """An iptables table""" + def __init__(self): self.rules = [] self.chains = set() def add_chain(self, name): + """Adds a named chain to the table + + The chain name is wrapped to be unique for the component creating + it, so different components of Nova can safely create identically + named chains without interfering with one another. + + At the moment, its wrapped name is -, + so if nova-compute creates a chain named "OUTPUT", it'll actually + end up named "nova-compute-OUTPUT". + """ self.chains.add(name) def remove_chain(self, name): + """Remove named chain + + This removal "cascades". All rule in the chain are removed, as are + all rules in other chains that jump to it. + + If the chain is not found, this is merely logged. + """ if name not in self.chains: LOG.debug(_("Attempted to remove chain %s which doesn't exist"), name) @@ -112,6 +136,15 @@ class IptablesTable(object): self.rules = filter(lambda r: jump_snippet not in r.rule, self.rules) def add_rule(self, chain, rule, wrap=True): + """Add a rule to the table + + This is just like what you'd feed to iptables, just without + the "-A " bit at the start. + + However, if you need to jump to one of your wrapped chains, + prepend its name with a '$' which will ensure the wrapping + is applied correctly. + """ if wrap and chain not in self.chains: raise ValueError(_("Unknown chain: %r") % chain) @@ -125,7 +158,13 @@ class IptablesTable(object): return '%s-%s' % (binary_name, s[1:]) return s - def remove_rule(self, *args, **kwargs): + def remove_rule(self, chain, rule, wrap=True): + """Remove a rule from a chain + + Note: The rule must be exactly identical to the one that was added. + You cannot switch arguments around like you can with the iptables + CLI tool. + """ try: self.rules.remove(IptablesRule(*args, **kwargs)) except ValueError: @@ -135,6 +174,22 @@ class IptablesTable(object): class IptablesManager(object): + """Wrapper for iptables + + See IptablesTable for some usage docs + + A number of chains are set up to begin with. + + For ipv4, the filter table has a INPUT, OUTPUT, FORWARD, and local chains + already set up, while the NAT chain has PREROUTING, OUTPUT, POSTROUTING, + and SNATTING. Except for "local" and "SNATTING" these are all set up so + that they are applied at the beginning of their non-wrapped counterparts. + "SNATTING" is jumped to from nat/POSTROUTING and "local" is jumped to from + filter/OUTPUT and filter/FORWARD. + + For ipv6, the filter table has INPUT, OUTPUT, FORWARD, and local. "local" + has the same semantics as for ipv4. + """ def __init__(self, execute=None): if not execute: if FLAGS.fake_network: @@ -190,6 +245,16 @@ class IptablesManager(object): self.semaphore = semaphore.Semaphore() def apply(self): + """Apply the current in-memory set of iptables rules + + This will blow away any rules left over from previous runs of the + same component of Nova, and replace them with our current set of + rules. This happens atomically, thanks to iptables-restore. + + We wrap the call in a semaphore lock, so that we don't race with + ourselves. In the event of a race with another component running + an iptables-* command at the same time, we retry up to 5 times. + """ with self.semaphore: s = [('iptables', self.ipv4)] if FLAGS.use_ipv6: @@ -200,14 +265,13 @@ class IptablesManager(object): current_table, _ = self.execute('sudo %s-save -t %s' % (cmd, table), attempts=5) current_lines = current_table.split('\n') - new_filter = self.modify_rules(current_lines, + new_filter = self._modify_rules(current_lines, tables[table]) self.execute('sudo %s-restore' % (cmd,), process_input='\n'.join(new_filter), attempts=5) - def modify_rules(self, current_lines, table, binary=None): - + def _modify_rules(self, current_lines, table, binary=None): chains = table.chains rules = table.rules -- cgit From 764f0a457e74c4498cbc9ea30a184e61f7932072 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Tue, 22 Feb 2011 13:18:21 +0900 Subject: just add 005_add_live_migration.py. --- .../versions/005_add_live_migration.py | 84 ++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py (limited to 'nova') diff --git a/nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py b/nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py new file mode 100644 index 000000000..903f7a646 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py @@ -0,0 +1,84 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +# +# New Tables +# + +compute_services = Table('compute_services', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('service_id', Integer(), nullable=False), + + Column('vcpus', Integer(), nullable=False), + Column('memory_mb', Integer(), nullable=False), + Column('local_gb', Integer(), nullable=False), + Column('vcpus_used', Integer(), nullable=False), + Column('memory_mb_used', Integer(), nullable=False), + Column('local_gb_used', Integer(), nullable=False), + Column('hypervisor_type', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=False), + Column('hypervisor_version', Integer(), nullable=False), + Column('cpu_info', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=False), + ) + + +# +# Tables to alter +# +instances_launched_on = Column( + 'launched_on', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=True) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + try: + compute_services.create() + except Exception: + logging.info(repr(compute_services)) + logging.exception('Exception while creating table') + meta.drop_all(tables=[compute_services]) + raise + + instances.create_column(instances_launched_on) -- cgit From c53bb1718a9b5900d09637d0ee966dadbf073900 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 09:00:29 +0100 Subject: Address some review comments. --- nova/network/linux_net.py | 16 ++++++++-------- nova/tests/test_virt.py | 1 - 2 files changed, 8 insertions(+), 9 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 3b6ec9338..42af73e74 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -54,8 +54,6 @@ flags.DEFINE_string('dhcpbridge', _bin_file('nova-dhcpbridge'), 'location of nova-dhcpbridge') flags.DEFINE_string('routing_source_ip', '$my_ip', 'Public IP of network host') -flags.DEFINE_bool('use_nova_chains', False, - 'use the nova_ routing chains instead of default') flags.DEFINE_string('input_chain', 'INPUT', 'chain to add nova_input to') @@ -266,7 +264,7 @@ class IptablesManager(object): (cmd, table), attempts=5) current_lines = current_table.split('\n') new_filter = self._modify_rules(current_lines, - tables[table]) + tables[table]) self.execute('sudo %s-restore' % (cmd,), process_input='\n'.join(new_filter), attempts=5) @@ -276,15 +274,17 @@ class IptablesManager(object): rules = table.rules # Remove any trace of our rules - new_filter = filter(lambda l: binary_name not in l, current_lines) + new_filter = filter(lambda line: binary_name not in line, + current_lines) seen_chains = False - for rules_index in range(len(new_filter)): + rules_index = 0 + for rules_index, rule in enumerate(new_filter): if not seen_chains: - if new_filter[rules_index].startswith(':'): + if rule.startswith(':'): seen_chains = True - elif seen_chains == 1: - if not new_filter[rules_index].startswith(':'): + else: + if not rule.startswith(':'): break new_filter[rules_index:rules_index] = [str(rule) for rule in rules] diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 11201788c..c2c7c8337 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -354,7 +354,6 @@ class IptablesFirewallTestCase(test.TestCase): instance_chain = None for rule in self.out_rules: - print rule # This is pretty crude, but it'll do for now if '-d 10.11.12.13 -j' in rule: instance_chain = rule.split(' ')[-1] -- cgit From 70d526dc44299b6bd54a5757d013ca3109887747 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 09:28:02 +0100 Subject: Add a new chain, floating-ip-snat, at the top of SNATTING, so that SNATting for floating ips gets applied before the default SNAT rule. --- nova/network/linux_net.py | 43 ++++++++++++++++++------------------------- 1 file changed, 18 insertions(+), 25 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 42af73e74..6b0735b5c 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -201,26 +201,13 @@ class IptablesManager(object): 'nat': IptablesTable()} self.ipv6 = {'filter': IptablesTable()} - self.ipv4['nat'].add_chain('SNATTING') - self.ipv4['nat'].add_rule('POSTROUTING', - '-j %s-SNATTING' % (binary_name,), - wrap=False) - self.ipv4['filter'].add_chain('local') - self.ipv4['filter'].add_rule('FORWARD', - '-j %s-local' % (binary_name,), - wrap=False) - self.ipv4['filter'].add_rule('OUTPUT', - '-j %s-local' % (binary_name,), - wrap=False) + self.ipv4['filter'].add_rule('FORWARD', '-j $local', wrap=False) + self.ipv4['filter'].add_rule('OUTPUT', '-j $local', wrap=False) self.ipv6['filter'].add_chain('local') - self.ipv6['filter'].add_rule('FORWARD', - '-j %s-local' % (binary_name,), - wrap=False) - self.ipv6['filter'].add_rule('OUTPUT', - '-j %s-local' % (binary_name,), - wrap=False) + self.ipv6['filter'].add_rule('FORWARD', '-j $local', wrap=False) + self.ipv6['filter'].add_rule('OUTPUT', '-j $local', wrap=False) # Wrap the builtin chains builtin_chains = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], @@ -236,12 +223,22 @@ class IptablesManager(object): for table, chains in builtin_chains[ip_version].iteritems(): for chain in chains: tables[table].add_chain(chain) - tables[table].add_rule(chain, - '-j %s-%s' % (binary_name, chain), + tables[table].add_rule(chain, '-j $%s' % (chain,), wrap=False) + # We add a SNATTING chain after our (wrapped) POSTROUTING chain + # so that rules added there will be applied after whatever we have in + # (the wrapped) POSTROUTING. + self.ipv4['nat'].add_chain('SNATTING') + self.ipv4['nat'].add_rule('POSTROUTING', '-j $SNATTING', wrap=False) + + self.ipv4['nat'].add_chain('floating-ip-snat') + self.ipv4['nat'].add_rule('SNATTING', '-j $floating-ip-snat') + self.semaphore = semaphore.Semaphore() + iptables_manager.apply() + def apply(self): """Apply the current in-memory set of iptables rules @@ -318,11 +315,6 @@ def init_host(): (FLAGS.fixed_range, FLAGS.routing_source_ip)) - iptables_manager.ipv4['nat'].add_rule("POSTROUTING", - "-s %s -j SNAT --to-source %s" % \ - (FLAGS.fixed_range, - FLAGS.routing_source_ip)) - iptables_manager.ipv4['nat'].add_rule("POSTROUTING", "-s %s -d %s -j ACCEPT" % \ (FLAGS.fixed_range, FLAGS.dmz_cidr)) @@ -377,7 +369,8 @@ def remove_floating_forward(floating_ip, fixed_ip): def floating_forward_rules(floating_ip, fixed_ip): return [("PREROUTING", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), ("OUTPUT", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), - ("SNATTING", "-d %s -j SNAT --to %s" % (fixed_ip, floating_ip))] + ("floating-ip-snat", + "-s %s -j SNAT --to %s" % (fixed_ip, floating_ip))] def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): -- cgit From bf37fb0ab5503a077a3d9e4109990d252e27cb15 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 11:29:58 +0100 Subject: Add a bunch of tests for everything. Add a 'head' kwarg to add_rule that lets the rule bubble to the top. This is needed for nova-filter-top to end up at the top. --- nova/network/linux_net.py | 120 ++++++++++++++++++++++++++++++------------ nova/tests/test_network.py | 128 +++++++++++++++++++++++++++++++++++++-------- 2 files changed, 192 insertions(+), 56 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 6b0735b5c..2ccb5969e 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -72,19 +72,22 @@ class IptablesRule(object): You shouldn't need to use this class directly, it's only used by IptablesManager """ - def __init__(self, chain, rule, wrap=True): + def __init__(self, chain, rule, wrap=True, head=False): self.chain = chain self.rule = rule self.wrap = wrap + self.head = head def __eq__(self, other): return ((self.chain == other.chain) and (self.rule == other.rule) and + (self.head == other.head) and (self.wrap == other.wrap)) def __ne__(self, other): return ((self.chain != other.chain) or (self.rule != other.rule) or + (self.head != other.head) or (self.wrap != other.wrap)) def __str__(self): @@ -101,8 +104,9 @@ class IptablesTable(object): def __init__(self): self.rules = [] self.chains = set() + self.unwrapped_chains = set() - def add_chain(self, name): + def add_chain(self, name, wrap=True): """Adds a named chain to the table The chain name is wrapped to be unique for the component creating @@ -113,9 +117,12 @@ class IptablesTable(object): so if nova-compute creates a chain named "OUTPUT", it'll actually end up named "nova-compute-OUTPUT". """ - self.chains.add(name) + if wrap: + self.chains.add(name) + else: + self.unwrapped_chains.add(name) - def remove_chain(self, name): + def remove_chain(self, name, wrap=True): """Remove named chain This removal "cascades". All rule in the chain are removed, as are @@ -123,17 +130,27 @@ class IptablesTable(object): If the chain is not found, this is merely logged. """ - if name not in self.chains: + if wrap: + chain_set = self.chains + else: + chain_set = self.unwrapped_chains + + if name not in chain_set: LOG.debug(_("Attempted to remove chain %s which doesn't exist"), name) return - self.chains.remove(name) + + chain_set.remove(name) self.rules = filter(lambda r: r.chain != name, self.rules) - jump_snippet = '-j %s-%s' % (binary_name, name) + if wrap: + jump_snippet = '-j %s-%s' % (binary_name, name) + else: + jump_snippet = '-j %s' % (name,) + self.rules = filter(lambda r: jump_snippet not in r.rule, self.rules) - def add_rule(self, chain, rule, wrap=True): + def add_rule(self, chain, rule, wrap=True, head=False): """Add a rule to the table This is just like what you'd feed to iptables, just without @@ -149,14 +166,14 @@ class IptablesTable(object): if '$' in rule: rule = ' '.join(map(self._wrap_target_chain, rule.split(' '))) - self.rules.append(IptablesRule(chain, rule, wrap)) + self.rules.append(IptablesRule(chain, rule, wrap, head)) def _wrap_target_chain(self, s): if s.startswith('$'): return '%s-%s' % (binary_name, s[1:]) return s - def remove_rule(self, chain, rule, wrap=True): + def remove_rule(self, chain, rule, wrap=True, head=False): """Remove a rule from a chain Note: The rule must be exactly identical to the one that was added. @@ -164,11 +181,12 @@ class IptablesTable(object): CLI tool. """ try: - self.rules.remove(IptablesRule(*args, **kwargs)) + self.rules.remove(IptablesRule(chain, rule, wrap, head)) except ValueError: LOG.debug(_("Tried to remove rule that wasn't there:" - " %(args)r %(kwargs)r"), {'args': args, - 'kwargs': kwargs}) + " %(chain)r %(rule)r %(wrap)r %(head)r"), + {'chain': chain, 'rule': rule, + 'head': head, 'wrap': wrap}) class IptablesManager(object): @@ -178,15 +196,19 @@ class IptablesManager(object): A number of chains are set up to begin with. - For ipv4, the filter table has a INPUT, OUTPUT, FORWARD, and local chains - already set up, while the NAT chain has PREROUTING, OUTPUT, POSTROUTING, - and SNATTING. Except for "local" and "SNATTING" these are all set up so - that they are applied at the beginning of their non-wrapped counterparts. - "SNATTING" is jumped to from nat/POSTROUTING and "local" is jumped to from - filter/OUTPUT and filter/FORWARD. + First, nova-filter-top. It's added at the top of FORWARD and OUTPUT. Its + name is not wrapped, so it's shared between the various nova workers. It's + intended for rules that need to live at the top of the FORWARD and OUTPUT + chains. It's in both the ipv4 and ipv6 set of tables. - For ipv6, the filter table has INPUT, OUTPUT, FORWARD, and local. "local" - has the same semantics as for ipv4. + For ipv4 and ipv6, the builtin INPUT, OUTPUT, and FORWARD filter chains are + wrapped, meaning that the "real" INPUT chain has a rule that jumps to the + wrapped INPUT chain, etc. Additionally, there's a wrapped chain named + "local" which is jumped to from nova-filter-top. + + For ipv4, the builtin PREROUTING, OUTPUT, and POSTROUTING nat chains are + wrapped in the same was as the builtin filter chains. Additionally, there's + a SNATTING chain that is applied after the POSTROUTING chain. """ def __init__(self, execute=None): if not execute: @@ -201,13 +223,19 @@ class IptablesManager(object): 'nat': IptablesTable()} self.ipv6 = {'filter': IptablesTable()} - self.ipv4['filter'].add_chain('local') - self.ipv4['filter'].add_rule('FORWARD', '-j $local', wrap=False) - self.ipv4['filter'].add_rule('OUTPUT', '-j $local', wrap=False) + # Add a nova-filter-top chain. It's intended to be shared + # among the various nova components. It sits at the very top + # of FORWARD and OUTPUT. + for tables in [self.ipv4, self.ipv6]: + tables['filter'].add_chain('nova-filter-top', wrap=False) + tables['filter'].add_rule('FORWARD', '-j nova-filter-top', + wrap=False, head=True) + tables['filter'].add_rule('OUTPUT', '-j nova-filter-top', + wrap=False, head=True) - self.ipv6['filter'].add_chain('local') - self.ipv6['filter'].add_rule('FORWARD', '-j $local', wrap=False) - self.ipv6['filter'].add_rule('OUTPUT', '-j $local', wrap=False) + tables['filter'].add_chain('local') + tables['filter'].add_rule('nova-filter-top', '-j $local', + wrap=False) # Wrap the builtin chains builtin_chains = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], @@ -226,18 +254,27 @@ class IptablesManager(object): tables[table].add_rule(chain, '-j $%s' % (chain,), wrap=False) - # We add a SNATTING chain after our (wrapped) POSTROUTING chain - # so that rules added there will be applied after whatever we have in - # (the wrapped) POSTROUTING. + # Add a nova-postrouting-bottom chain. It's intended to be shared + # among the various nova components. We set it as the last chain + # of POSTROUTING chain. + self.ipv4['nat'].add_chain('nova-postrouting-bottom', wrap=False) + self.ipv4['nat'].add_rule('POSTROUTING', '-j nova-postrouting-bottom', + wrap=False) + + + # We add a SNATTING chain to the shared nova-postrouting-bottom chain + # so that it's applied last. self.ipv4['nat'].add_chain('SNATTING') - self.ipv4['nat'].add_rule('POSTROUTING', '-j $SNATTING', wrap=False) + self.ipv4['nat'].add_rule('nova-postrouting-bottom', '-j $SNATTING', wrap=False) + # And then we add a floating-ip-snat chain and jump to first thing in the SNATTING + # chain. self.ipv4['nat'].add_chain('floating-ip-snat') self.ipv4['nat'].add_rule('SNATTING', '-j $floating-ip-snat') self.semaphore = semaphore.Semaphore() - iptables_manager.apply() + self.apply() def apply(self): """Apply the current in-memory set of iptables rules @@ -245,7 +282,7 @@ class IptablesManager(object): This will blow away any rules left over from previous runs of the same component of Nova, and replace them with our current set of rules. This happens atomically, thanks to iptables-restore. - + We wrap the call in a semaphore lock, so that we don't race with ourselves. In the event of a race with another component running an iptables-* command at the same time, we retry up to 5 times. @@ -267,6 +304,7 @@ class IptablesManager(object): attempts=5) def _modify_rules(self, current_lines, table, binary=None): + unwrapped_chains = table.unwrapped_chains chains = table.chains rules = table.rules @@ -284,11 +322,25 @@ class IptablesManager(object): if not rule.startswith(':'): break - new_filter[rules_index:rules_index] = [str(rule) for rule in rules] + new_filter[rules_index:rules_index] = [str(rule) for rule in rules + if rule.head or + str(rule) not in new_filter] + new_filter[rules_index:rules_index] = [':%s - [0:0]' % \ + (name,) \ + for name in unwrapped_chains] new_filter[rules_index:rules_index] = [':%s-%s - [0:0]' % \ (binary_name, name,) \ for name in chains] + seen_lines = set() + def _weed_out_duplicates(line): + if line in seen_lines: + return False + else: + seen_lines.add(line) + return True + + new_filter = filter(_weed_out_duplicates, new_filter) return new_filter diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index f1d4fe133..afd38272d 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -36,12 +36,22 @@ LOG = logging.getLogger('nova.tests.network') class IptablesManagerTestCase(test.TestCase): - sample_filter = """# Completed on Fri Feb 18 15:17:05 2011 -# Generated by iptables-save v1.4.10 on Fri Feb 18 15:17:05 2011 + sample_filter = """# Generated by iptables-save on Fri Feb 18 15:17:05 2011 *filter :INPUT ACCEPT [2223527:305688874] :FORWARD ACCEPT [0:0] :OUTPUT ACCEPT [2172501:140856656] +:nova-compute-FORWARD - [0:0] +:nova-compute-INPUT - [0:0] +:nova-compute-local - [0:0] +:nova-compute-OUTPUT - [0:0] +:nova-filter-top - [0:0] +-A FORWARD -j nova-filter-top +-A OUTPUT -j nova-filter-top +-A nova-filter-top -j nova-compute-local +-A INPUT -j nova-compute-INPUT +-A OUTPUT -j nova-compute-OUTPUT +-A FORWARD -j nova-compute-FORWARD -A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT -A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT -A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT @@ -53,42 +63,116 @@ class IptablesManagerTestCase(test.TestCase): COMMIT # Completed on Fri Feb 18 15:17:05 2011""" + sample_nat = """# Generated by iptables-save on Fri Feb 18 15:17:05 2011 +*nat +:PREROUTING ACCEPT [3936:762355] +:INPUT ACCEPT [2447:225266] +:OUTPUT ACCEPT [63491:4191863] +:POSTROUTING ACCEPT [63112:4108641] +:nova-compute-OUTPUT - [0:0] +:nova-compute-floating-ip-snat - [0:0] +:nova-compute-SNATTING - [0:0] +:nova-compute-PREROUTING - [0:0] +:nova-compute-POSTROUTING - [0:0] +:nova-postrouting-bottom - [0:0] +-A PREROUTING -j nova-compute-PREROUTING +-A OUTPUT -j nova-compute-OUTPUT +-A POSTROUTING -j nova-compute-POSTROUTING +-A POSTROUTING -j nova-postrouting-bottom +-A nova-postrouting-bottom -j nova-compute-SNATTING +-A nova-compute-SNATTING -j nova-compute-floating-ip-snat +COMMIT +# Completed on Fri Feb 18 15:17:05 2011 +""" + def setUp(self): super(IptablesManagerTestCase, self).setUp() self.manager = linux_net.IptablesManager() - def test_rules_are_wrapped(self): + + def test_filter_rules_are_wrapped(self): current_lines = self.sample_filter.split('\n') table = self.manager.ipv4['filter'] table.add_rule('FORWARD', '-s 1.2.3.4/5 -j DROP') - new_lines = self.manager.modify_rules(current_lines, table) + new_lines = self.manager._modify_rules(current_lines, table) self.assertTrue('-A run_tests.py-FORWARD ' '-s 1.2.3.4/5 -j DROP' in new_lines) table.remove_rule('FORWARD', '-s 1.2.3.4/5 -j DROP') - new_lines = self.manager.modify_rules(current_lines, table) + new_lines = self.manager._modify_rules(current_lines, table) self.assertTrue('-A run_tests.py-FORWARD ' '-s 1.2.3.4/5 -j DROP' not in new_lines) - def test_wrapper_rules_in_place(self): - current_lines = self.sample_filter.split('\n') + def test_nat_rules(self): + current_lines = self.sample_nat.split('\n') + new_lines = self.manager._modify_rules(current_lines, + self.manager.ipv4['nat']) + + for line in [':nova-compute-OUTPUT - [0:0]', + ':nova-compute-floating-ip-snat - [0:0]', + ':nova-compute-SNATTING - [0:0]', + ':nova-compute-PREROUTING - [0:0]', + ':nova-compute-POSTROUTING - [0:0]']: + self.assertTrue(line in new_lines, "One of nova-compute's chains " + "went missing.") + + seen_lines = set() + for line in new_lines: + self.assertTrue(line not in seen_lines, + "Duplicate line: %s" % line) + seen_lines.add(line) - # TODO(soren): Add stuff for ipv6 - check_matrix = {4: {'filter': ['INPUT', 'OUTPUT', 'FORWARD'], - 'nat': ['PREROUTING', 'OUTPUT', 'POSTROUTING']}, - 6: {'filter': ['INPUT', 'OUTPUT', 'FORWARD']}} - - for ip_version in check_matrix: - ip = getattr(self.manager, 'ipv%d' % ip_version) - for table_name in ip: - table = ip[table_name] - new_lines = self.manager.modify_rules(current_lines, table) - for chain in check_matrix[ip_version][table_name]: - self.assertTrue(':run_tests.py-%s - [0:0]' % \ - (chain,) in new_lines) - self.assertTrue('-A %s -j run_tests.py-%s' % \ - (chain, chain) in new_lines) + last_postrouting_line = '' + + for line in new_lines: + if line.startswith('-A POSTROUTING'): + last_postrouting_line = line + + self.assertTrue('-j nova-postrouting-bottom' in last_postrouting_line, + "Last POSTROUTING rule does not jump to " + "nova-postouting-bottom: %s" % last_postrouting_line) + + for chain in ['POSTROUTING', 'PREROUTING', 'OUTPUT']: + self.assertTrue('-A %s -j run_tests.py-%s' \ + % (chain, chain) in new_lines, + "Built-in chain %s not wrapped" % (chain,)) + + + def test_filter_rules(self): + current_lines = self.sample_filter.split('\n') + new_lines = self.manager._modify_rules(current_lines, + self.manager.ipv4['filter']) + + for line in [':nova-compute-FORWARD - [0:0]', + ':nova-compute-INPUT - [0:0]', + ':nova-compute-local - [0:0]', + ':nova-compute-OUTPUT - [0:0]']: + self.assertTrue(line in new_lines, "One of nova-compute's chains" + " went missing.") + + seen_lines = set() + for line in new_lines: + self.assertTrue(line not in seen_lines, + "Duplicate line: %s" % line) + seen_lines.add(line) + + for chain in ['FORWARD', 'OUTPUT']: + for line in new_lines: + if line.startswith('-A %s' % chain): + self.assertTrue('-j nova-filter-top' in line, + "First %s rule does not " + "jump to nova-filter-top" % chain) + break + + self.assertTrue('-A nova-filter-top ' + '-j run_tests.py-local' in new_lines, + "nova-filter-top does not jump to wrapped local chain") + + for chain in ['INPUT', 'OUTPUT', 'FORWARD']: + self.assertTrue('-A %s -j run_tests.py-%s' \ + % (chain, chain) in new_lines, + "Built-in chain %s not wrapped" % (chain,)) class NetworkTestCase(test.TestCase): -- cgit From 54f2362d09393ad6ccdfee5689d4f547c69b3f42 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 11:47:46 +0100 Subject: Remove leftover from debugging. --- nova/network/linux_net.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 2ccb5969e..5f480f633 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -274,8 +274,6 @@ class IptablesManager(object): self.semaphore = semaphore.Semaphore() - self.apply() - def apply(self): """Apply the current in-memory set of iptables rules -- cgit From b5e6601f76d64a96d6c7de5e9acdf5a8cf0fe8e9 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 12:21:29 +0100 Subject: PEP8 adjustments. --- nova/network/linux_net.py | 9 +++++---- nova/tests/test_network.py | 10 ++++------ 2 files changed, 9 insertions(+), 10 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 5f480f633..7c4c16810 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -261,14 +261,14 @@ class IptablesManager(object): self.ipv4['nat'].add_rule('POSTROUTING', '-j nova-postrouting-bottom', wrap=False) - # We add a SNATTING chain to the shared nova-postrouting-bottom chain # so that it's applied last. self.ipv4['nat'].add_chain('SNATTING') - self.ipv4['nat'].add_rule('nova-postrouting-bottom', '-j $SNATTING', wrap=False) + self.ipv4['nat'].add_rule('nova-postrouting-bottom', '-j $SNATTING', + wrap=False) - # And then we add a floating-ip-snat chain and jump to first thing in the SNATTING - # chain. + # And then we add a floating-ip-snat chain and jump to first thing in + # the SNATTING chain. self.ipv4['nat'].add_chain('floating-ip-snat') self.ipv4['nat'].add_rule('SNATTING', '-j $floating-ip-snat') @@ -331,6 +331,7 @@ class IptablesManager(object): for name in chains] seen_lines = set() + def _weed_out_duplicates(line): if line in seen_lines: return False diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index afd38272d..2bdf3709e 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -89,7 +89,6 @@ COMMIT super(IptablesManagerTestCase, self).setUp() self.manager = linux_net.IptablesManager() - def test_filter_rules_are_wrapped(self): current_lines = self.sample_filter.split('\n') @@ -114,8 +113,8 @@ COMMIT ':nova-compute-SNATTING - [0:0]', ':nova-compute-PREROUTING - [0:0]', ':nova-compute-POSTROUTING - [0:0]']: - self.assertTrue(line in new_lines, "One of nova-compute's chains " - "went missing.") + self.assertTrue(line in new_lines, "One of nova-compute's chains " + "went missing.") seen_lines = set() for line in new_lines: @@ -138,7 +137,6 @@ COMMIT % (chain, chain) in new_lines, "Built-in chain %s not wrapped" % (chain,)) - def test_filter_rules(self): current_lines = self.sample_filter.split('\n') new_lines = self.manager._modify_rules(current_lines, @@ -148,8 +146,8 @@ COMMIT ':nova-compute-INPUT - [0:0]', ':nova-compute-local - [0:0]', ':nova-compute-OUTPUT - [0:0]']: - self.assertTrue(line in new_lines, "One of nova-compute's chains" - " went missing.") + self.assertTrue(line in new_lines, "One of nova-compute's chains" + " went missing.") seen_lines = set() for line in new_lines: -- cgit From 92a693b04feddad2420a835a12f53520c5529d8f Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 12:42:38 +0100 Subject: floating-ip-snat was too long. Use floating-snat instead. --- nova/network/linux_net.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 7c4c16810..5e89a1df6 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -267,10 +267,10 @@ class IptablesManager(object): self.ipv4['nat'].add_rule('nova-postrouting-bottom', '-j $SNATTING', wrap=False) - # And then we add a floating-ip-snat chain and jump to first thing in + # And then we add a floating-snat chain and jump to first thing in # the SNATTING chain. - self.ipv4['nat'].add_chain('floating-ip-snat') - self.ipv4['nat'].add_rule('SNATTING', '-j $floating-ip-snat') + self.ipv4['nat'].add_chain('floating-snat') + self.ipv4['nat'].add_rule('SNATTING', '-j $floating-snat') self.semaphore = semaphore.Semaphore() @@ -420,7 +420,7 @@ def remove_floating_forward(floating_ip, fixed_ip): def floating_forward_rules(floating_ip, fixed_ip): return [("PREROUTING", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), ("OUTPUT", "-d %s -j DNAT --to %s" % (floating_ip, fixed_ip)), - ("floating-ip-snat", + ("floating-snat", "-s %s -j SNAT --to %s" % (fixed_ip, floating_ip))] -- cgit From 443f01ef7d977ba6ff86508b908f66733bdd989e Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 14:32:20 +0100 Subject: Account for the fact that iptables-save outputs rules with a space at the end. Reverse the rule deduplication so that the last one takes precedence. --- nova/network/linux_net.py | 45 ++++++++++++++++++++++++++++++--------------- nova/tests/test_network.py | 42 ++++++++++++++++++++++-------------------- 2 files changed, 52 insertions(+), 35 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 5e89a1df6..849181641 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -72,22 +72,22 @@ class IptablesRule(object): You shouldn't need to use this class directly, it's only used by IptablesManager """ - def __init__(self, chain, rule, wrap=True, head=False): + def __init__(self, chain, rule, wrap=True, top=False): self.chain = chain self.rule = rule self.wrap = wrap - self.head = head + self.top = top def __eq__(self, other): return ((self.chain == other.chain) and (self.rule == other.rule) and - (self.head == other.head) and + (self.top == other.top) and (self.wrap == other.wrap)) def __ne__(self, other): return ((self.chain != other.chain) or (self.rule != other.rule) or - (self.head != other.head) or + (self.top != other.top) or (self.wrap != other.wrap)) def __str__(self): @@ -150,7 +150,7 @@ class IptablesTable(object): self.rules = filter(lambda r: jump_snippet not in r.rule, self.rules) - def add_rule(self, chain, rule, wrap=True, head=False): + def add_rule(self, chain, rule, wrap=True, top=False): """Add a rule to the table This is just like what you'd feed to iptables, just without @@ -166,14 +166,14 @@ class IptablesTable(object): if '$' in rule: rule = ' '.join(map(self._wrap_target_chain, rule.split(' '))) - self.rules.append(IptablesRule(chain, rule, wrap, head)) + self.rules.append(IptablesRule(chain, rule, wrap, top)) def _wrap_target_chain(self, s): if s.startswith('$'): return '%s-%s' % (binary_name, s[1:]) return s - def remove_rule(self, chain, rule, wrap=True, head=False): + def remove_rule(self, chain, rule, wrap=True, top=False): """Remove a rule from a chain Note: The rule must be exactly identical to the one that was added. @@ -181,12 +181,12 @@ class IptablesTable(object): CLI tool. """ try: - self.rules.remove(IptablesRule(chain, rule, wrap, head)) + self.rules.remove(IptablesRule(chain, rule, wrap, top)) except ValueError: LOG.debug(_("Tried to remove rule that wasn't there:" - " %(chain)r %(rule)r %(wrap)r %(head)r"), + " %(chain)r %(rule)r %(wrap)r %(top)r"), {'chain': chain, 'rule': rule, - 'head': head, 'wrap': wrap}) + 'top': top, 'wrap': wrap}) class IptablesManager(object): @@ -229,9 +229,9 @@ class IptablesManager(object): for tables in [self.ipv4, self.ipv6]: tables['filter'].add_chain('nova-filter-top', wrap=False) tables['filter'].add_rule('FORWARD', '-j nova-filter-top', - wrap=False, head=True) + wrap=False, top=True) tables['filter'].add_rule('OUTPUT', '-j nova-filter-top', - wrap=False, head=True) + wrap=False, top=True) tables['filter'].add_chain('local') tables['filter'].add_rule('nova-filter-top', '-j $local', @@ -320,9 +320,19 @@ class IptablesManager(object): if not rule.startswith(':'): break - new_filter[rules_index:rules_index] = [str(rule) for rule in rules - if rule.head or - str(rule) not in new_filter] + our_rules = [] + for rule in rules: + rule_str = str(rule) + if rule.top: + # rule.top == True means we want this rule to be at the top. + # Further down, we weed out duplicates from the bottom of the + # list, so here we remove the dupes ahead of time. + new_filter = filter(lambda s: s.strip() != rule_str.strip(), + new_filter) + our_rules += [rule_str] + + new_filter[rules_index:rules_index] = our_rules + new_filter[rules_index:rules_index] = [':%s - [0:0]' % \ (name,) \ for name in unwrapped_chains] @@ -333,13 +343,18 @@ class IptablesManager(object): seen_lines = set() def _weed_out_duplicates(line): + line = line.strip() if line in seen_lines: return False else: seen_lines.add(line) return True + # We filter duplicates, letting the *last* occurrence take + # precendence. + new_filter.reverse() new_filter = filter(_weed_out_duplicates, new_filter) + new_filter.reverse() return new_filter diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 2bdf3709e..b1d70e323 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -46,20 +46,20 @@ class IptablesManagerTestCase(test.TestCase): :nova-compute-local - [0:0] :nova-compute-OUTPUT - [0:0] :nova-filter-top - [0:0] --A FORWARD -j nova-filter-top --A OUTPUT -j nova-filter-top --A nova-filter-top -j nova-compute-local --A INPUT -j nova-compute-INPUT --A OUTPUT -j nova-compute-OUTPUT --A FORWARD -j nova-compute-FORWARD --A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT --A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT --A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT --A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT --A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT --A FORWARD -i virbr0 -o virbr0 -j ACCEPT --A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable --A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable +-A FORWARD -j nova-filter-top +-A OUTPUT -j nova-filter-top +-A nova-filter-top -j nova-compute-local +-A INPUT -j nova-compute-INPUT +-A OUTPUT -j nova-compute-OUTPUT +-A FORWARD -j nova-compute-FORWARD +-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT +-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT +-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT +-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT +-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT +-A FORWARD -i virbr0 -o virbr0 -j ACCEPT +-A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable +-A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable COMMIT # Completed on Fri Feb 18 15:17:05 2011""" @@ -75,12 +75,12 @@ COMMIT :nova-compute-PREROUTING - [0:0] :nova-compute-POSTROUTING - [0:0] :nova-postrouting-bottom - [0:0] --A PREROUTING -j nova-compute-PREROUTING --A OUTPUT -j nova-compute-OUTPUT --A POSTROUTING -j nova-compute-POSTROUTING --A POSTROUTING -j nova-postrouting-bottom --A nova-postrouting-bottom -j nova-compute-SNATTING --A nova-compute-SNATTING -j nova-compute-floating-ip-snat +-A PREROUTING -j nova-compute-PREROUTING +-A OUTPUT -j nova-compute-OUTPUT +-A POSTROUTING -j nova-compute-POSTROUTING +-A POSTROUTING -j nova-postrouting-bottom +-A nova-postrouting-bottom -j nova-compute-SNATTING +-A nova-compute-SNATTING -j nova-compute-floating-ip-snat COMMIT # Completed on Fri Feb 18 15:17:05 2011 """ @@ -118,6 +118,7 @@ COMMIT seen_lines = set() for line in new_lines: + line = line.strip() self.assertTrue(line not in seen_lines, "Duplicate line: %s" % line) seen_lines.add(line) @@ -151,6 +152,7 @@ COMMIT seen_lines = set() for line in new_lines: + line = line.strip() self.assertTrue(line not in seen_lines, "Duplicate line: %s" % line) seen_lines.add(line) -- cgit From 7eee81ee6480a36b179ae26be88ebad9415c4b62 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 14:40:00 +0100 Subject: PEP8 again --- nova/tests/test_network.py | 103 +++++++++++++++++++++++---------------------- 1 file changed, 52 insertions(+), 51 deletions(-) (limited to 'nova') diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index b1d70e323..d3a23abf4 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -36,61 +36,62 @@ LOG = logging.getLogger('nova.tests.network') class IptablesManagerTestCase(test.TestCase): - sample_filter = """# Generated by iptables-save on Fri Feb 18 15:17:05 2011 -*filter -:INPUT ACCEPT [2223527:305688874] -:FORWARD ACCEPT [0:0] -:OUTPUT ACCEPT [2172501:140856656] -:nova-compute-FORWARD - [0:0] -:nova-compute-INPUT - [0:0] -:nova-compute-local - [0:0] -:nova-compute-OUTPUT - [0:0] -:nova-filter-top - [0:0] --A FORWARD -j nova-filter-top --A OUTPUT -j nova-filter-top --A nova-filter-top -j nova-compute-local --A INPUT -j nova-compute-INPUT --A OUTPUT -j nova-compute-OUTPUT --A FORWARD -j nova-compute-FORWARD --A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT --A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT --A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT --A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT --A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT --A FORWARD -i virbr0 -o virbr0 -j ACCEPT --A FORWARD -o virbr0 -j REJECT --reject-with icmp-port-unreachable --A FORWARD -i virbr0 -j REJECT --reject-with icmp-port-unreachable -COMMIT -# Completed on Fri Feb 18 15:17:05 2011""" - - sample_nat = """# Generated by iptables-save on Fri Feb 18 15:17:05 2011 -*nat -:PREROUTING ACCEPT [3936:762355] -:INPUT ACCEPT [2447:225266] -:OUTPUT ACCEPT [63491:4191863] -:POSTROUTING ACCEPT [63112:4108641] -:nova-compute-OUTPUT - [0:0] -:nova-compute-floating-ip-snat - [0:0] -:nova-compute-SNATTING - [0:0] -:nova-compute-PREROUTING - [0:0] -:nova-compute-POSTROUTING - [0:0] -:nova-postrouting-bottom - [0:0] --A PREROUTING -j nova-compute-PREROUTING --A OUTPUT -j nova-compute-OUTPUT --A POSTROUTING -j nova-compute-POSTROUTING --A POSTROUTING -j nova-postrouting-bottom --A nova-postrouting-bottom -j nova-compute-SNATTING --A nova-compute-SNATTING -j nova-compute-floating-ip-snat -COMMIT -# Completed on Fri Feb 18 15:17:05 2011 -""" + sample_filter = ['#Generated by iptables-save on Fri Feb 18 15:17:05 2011', + '*filter', + ':INPUT ACCEPT [2223527:305688874]', + ':FORWARD ACCEPT [0:0]', + ':OUTPUT ACCEPT [2172501:140856656]', + ':nova-compute-FORWARD - [0:0]', + ':nova-compute-INPUT - [0:0]', + ':nova-compute-local - [0:0]', + ':nova-compute-OUTPUT - [0:0]', + ':nova-filter-top - [0:0]', + '-A FORWARD -j nova-filter-top ', + '-A OUTPUT -j nova-filter-top ', + '-A nova-filter-top -j nova-compute-local ', + '-A INPUT -j nova-compute-INPUT ', + '-A OUTPUT -j nova-compute-OUTPUT ', + '-A FORWARD -j nova-compute-FORWARD ', + '-A INPUT -i virbr0 -p udp -m udp --dport 53 -j ACCEPT ', + '-A INPUT -i virbr0 -p tcp -m tcp --dport 53 -j ACCEPT ', + '-A INPUT -i virbr0 -p udp -m udp --dport 67 -j ACCEPT ', + '-A INPUT -i virbr0 -p tcp -m tcp --dport 67 -j ACCEPT ', + '-A FORWARD -s 192.168.122.0/24 -i virbr0 -j ACCEPT ', + '-A FORWARD -i virbr0 -o virbr0 -j ACCEPT ', + '-A FORWARD -o virbr0 -j REJECT --reject-with ' + 'icmp-port-unreachable ', + '-A FORWARD -i virbr0 -j REJECT --reject-with ' + 'icmp-port-unreachable ', + 'COMMIT', + '# Completed on Fri Feb 18 15:17:05 2011'] + + sample_nat = ['# Generated by iptables-save on Fri Feb 18 15:17:05 2011', + '*nat', + ':PREROUTING ACCEPT [3936:762355]', + ':INPUT ACCEPT [2447:225266]', + ':OUTPUT ACCEPT [63491:4191863]', + ':POSTROUTING ACCEPT [63112:4108641]', + ':nova-compute-OUTPUT - [0:0]', + ':nova-compute-floating-ip-snat - [0:0]', + ':nova-compute-SNATTING - [0:0]', + ':nova-compute-PREROUTING - [0:0]', + ':nova-compute-POSTROUTING - [0:0]', + ':nova-postrouting-bottom - [0:0]', + '-A PREROUTING -j nova-compute-PREROUTING ', + '-A OUTPUT -j nova-compute-OUTPUT ', + '-A POSTROUTING -j nova-compute-POSTROUTING ', + '-A POSTROUTING -j nova-postrouting-bottom ', + '-A nova-postrouting-bottom -j nova-compute-SNATTING ', + '-A nova-compute-SNATTING -j nova-compute-floating-ip-snat ', + 'COMMIT', + '# Completed on Fri Feb 18 15:17:05 2011'] def setUp(self): super(IptablesManagerTestCase, self).setUp() self.manager = linux_net.IptablesManager() def test_filter_rules_are_wrapped(self): - current_lines = self.sample_filter.split('\n') + current_lines = self.sample_filter table = self.manager.ipv4['filter'] table.add_rule('FORWARD', '-s 1.2.3.4/5 -j DROP') @@ -104,7 +105,7 @@ COMMIT '-s 1.2.3.4/5 -j DROP' not in new_lines) def test_nat_rules(self): - current_lines = self.sample_nat.split('\n') + current_lines = self.sample_nat new_lines = self.manager._modify_rules(current_lines, self.manager.ipv4['nat']) @@ -139,7 +140,7 @@ COMMIT "Built-in chain %s not wrapped" % (chain,)) def test_filter_rules(self): - current_lines = self.sample_filter.split('\n') + current_lines = self.sample_filter new_lines = self.manager._modify_rules(current_lines, self.manager.ipv4['filter']) -- cgit From c32e57999be09368b18f5a89315465e629ed4819 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Tue, 22 Feb 2011 23:55:03 +0900 Subject: Fixed based on reviewer's comment. 1. Change docstrings format 2. Fix comment grammer mistake, etc --- nova/compute/manager.py | 123 +++++--- nova/db/api.py | 12 +- nova/db/sqlalchemy/api.py | 27 +- .../versions/005_add_live_migration.py | 3 +- nova/db/sqlalchemy/models.py | 12 +- nova/scheduler/driver.py | 121 +++++--- nova/scheduler/manager.py | 15 +- nova/tests/test_compute.py | 77 ++--- nova/tests/test_scheduler.py | 141 ++++------ nova/tests/test_service.py | 6 +- nova/tests/test_virt.py | 309 +++++++++------------ nova/virt/fake.py | 74 +---- nova/virt/libvirt_conn.py | 182 ++++++++---- nova/virt/xenapi_conn.py | 14 +- nova/volume/driver.py | 14 +- 15 files changed, 557 insertions(+), 573 deletions(-) (limited to 'nova') diff --git a/nova/compute/manager.py b/nova/compute/manager.py index d548cef6f..5b6e9082e 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -36,10 +36,10 @@ terminating it. import base64 import datetime +import os import random import string import socket -import os import tempfile import time import functools @@ -65,8 +65,8 @@ flags.DEFINE_string('console_host', socket.gethostname(), 'Console proxy host to use to connect to instances on' 'this host.') flags.DEFINE_string('live_migration_retry_count', 30, - ("""Retry count needed in live_migration.""" - """ sleep 1 sec for each count""")) + ("Retry count needed in live_migration." + " sleep 1 sec for each count")) LOG = logging.getLogger('nova.compute.manager') @@ -602,31 +602,66 @@ class ComputeManager(manager.Manager): @exception.wrap_exception def compare_cpu(self, context, cpu_info): - """ Check the host cpu is compatible to a cpu given by xml.""" + """Checks the host cpu is compatible to a cpu given by xml. + + :param context: security context + :param cpu_info: json string obtained from virConnect.getCapabilities + :returns: See driver.compare_cpu + + """ return self.driver.compare_cpu(cpu_info) @exception.wrap_exception def mktmpfile(self, context): - """make tmpfile under FLAGS.instance_path.""" - fd, name = tempfile.mkstemp(dir=FLAGS.instances_path) - # No essential reason to write dateinfo. just for debugging reason. - os.fdopen(fd, 'w').write(str(datetime.datetime.utcnow())) + """Makes tmpfile under FLAGS.instance_path. + + This method enables compute nodes to recognize that they mounts + same shared storage. mktmpfile()/confirm_tmpfile is a pair. + + :param context: security context + :returns: tmpfile name + + """ + + dirpath = FLAGS.instances_path + fd, name = tempfile.mkstemp(dir=dirpath) + LOG.debug(_("Creating tmpfile %s to notify to other " + "compute node that they mounts same storage.") % name) + os.fdopen(fd, 'w+').close() return name @exception.wrap_exception def confirm_tmpfile(self, context, path): - """Confirm existence of the tmpfile given by path.""" + """Confirms existence of the tmpfile given by path. + + :param context: security context + :param path: confirm existence of this path + :returns: depends on os.remove() + + """ + if not os.path.exists(path): raise exception.NotFound(_('%s not found') % path) return os.remove(path) @exception.wrap_exception def update_available_resource(self, context): - """See comments update_resource_info""" + """See comments update_resource_info. + + :param context: security context + :returns: See driver.update_available_resource() + + """ + return self.driver.update_available_resource(context, self.host) def pre_live_migration(self, context, instance_id): - """Any preparation for live migration at dst host.""" + """Preparations for live migration at dest host. + + :param context: security context + :param instance_id: nova.db.sqlalchemy.models.Instance.Id + + """ # Getting instance info instance_ref = self.db.instance_get(context, instance_id) @@ -635,7 +670,7 @@ class ComputeManager(manager.Manager): # Getting fixed ips fixed_ip = self.db.instance_get_fixed_address(context, instance_id) if not fixed_ip: - msg = _("%(instance_id)s(%(ec2_id)s) doesnt have fixed_ip") + msg = _("%(instance_id)s(%(ec2_id)s) does'nt have fixed_ip") raise exception.NotFound(msg % locals()) # If any volume is mounted, prepare here. @@ -645,8 +680,8 @@ class ComputeManager(manager.Manager): for v in instance_ref['volumes']: self.volume_manager.setup_compute_volume(context, v['id']) - # Bridge settings - # call this method prior to ensure_filtering_rules_for_instance, + # Bridge settings. + # Call this method prior to ensure_filtering_rules_for_instance, # since bridge is not set up, ensure_filtering_rules_for instance # fails. # @@ -660,24 +695,29 @@ class ComputeManager(manager.Manager): break except exception.ProcessExecutionError, e: if i == max_retry - 1: - raise e + raise else: - LOG.warn(_("setup_compute_network() fail %(i)d th. " - "Retry up to %(max_retry)d for %(ec2_id)s") + LOG.warn(_("setup_compute_network() failed %(i)d." + "Retry up to %(max_retry)d for %(ec2_id)s.") % locals()) time.sleep(1) # Creating filters to hypervisors and firewalls. # An example is that nova-instance-instance-xxx, - # which is written to libvirt.xml( check "virsh nwfilter-list ) - # On destination host, this nwfilter is necessary. + # which is written to libvirt.xml(Check "virsh nwfilter-list") + # This nwfilter is necessary on the destination host. # In addition, this method is creating filtering rule # onto destination host. self.driver.ensure_filtering_rules_for_instance(instance_ref) - #@exception.wrap_exception def live_migration(self, context, instance_id, dest): - """Executing live migration.""" + """Executing live migration. + + :param context: security context + :param instance_id: nova.db.sqlalchemy.models.Instance.Id + :param dest: destination host + + """ # Get instance for error handling. instance_ref = self.db.instance_get(context, instance_id) @@ -702,7 +742,7 @@ class ComputeManager(manager.Manager): msg = _("Pre live migration for %(i_name)s failed at %(dest)s") LOG.error(msg % locals()) self.recover_live_migration(context, instance_ref) - raise e + raise # Executing live migration # live_migration might raises exceptions, but @@ -712,10 +752,17 @@ class ComputeManager(manager.Manager): self.recover_live_migration) def post_live_migration(self, ctxt, instance_ref, dest): + """Post operations for live migration. + + This method is called from live_migration + and mainly updating database record. + + :param ctxt: security context + :param instance_id: nova.db.sqlalchemy.models.Instance.Id + :param dest: destination host + """ - Post operations for live migration. - Mainly, database updating. - """ + LOG.info(_('post_live_migration() is started..')) instance_id = instance_ref['id'] @@ -734,19 +781,12 @@ class ComputeManager(manager.Manager): # Database updating. i_name = instance_ref.name - #fixed_ip = self.db.instance_get_fixed_address(ctxt, instance_id) - # Not return if fixed_ip is not found, otherwise, - # instance never be accessible.. - #if None == fixed_ip: - # LOG.warn(_('fixed_ip is not found for %s.') % i_name) - #self.db.fixed_ip_update(ctxt, fixed_ip, {'host': dest}) - try: # Not return if floating_ip is not found, otherwise, # instance never be accessible.. floating_ip = self.db.instance_get_floating_address(ctxt, instance_id) - if None == floating_ip: + if not floating_ip: LOG.info(_('floating_ip is not found for %s'), i_name) else: floating_ip_ref = self.db.floating_ip_get_by_address(ctxt, @@ -763,15 +803,23 @@ class ComputeManager(manager.Manager): # Restore instance/volume state self.recover_live_migration(ctxt, instance_ref, dest) - msg = _('Migrating %(i_name)s to %(dest)s finishes successfully.') - LOG.info(msg % locals()) + LOG.info(_('Migrating %(i_name)s to %(dest)s finishes successfully.') + % locals()) LOG.info(_("The below error is normally occurs." "Just check if instance is successfully migrated.\n" "libvir: QEMU error : Domain not found: no domain " "with matching name..")) def recover_live_migration(self, ctxt, instance_ref, host=None): - """Instance/volume state is recovered from migrating -> running.""" + """Recovers Instance/volume state from migrating -> running. + + :param ctxt: security context + :param instance_id: nova.db.sqlalchemy.models.Instance.Id + :param host: + DB column value is updated by this hostname. + if none, the host instance currently running is selected. + + """ if not host: host = instance_ref['host'] @@ -783,5 +831,4 @@ class ComputeManager(manager.Manager): 'host': host}) for v in instance_ref['volumes']: - self.db.volume_update(ctxt, v['id'], {'status': 'in-use', - 'host': host}) + self.db.volume_update(ctxt, v['id'], {'status': 'in-use'}) diff --git a/nova/db/api.py b/nova/db/api.py index 609f62495..e10a06178 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -169,6 +169,7 @@ def compute_service_update(context, compute_id, values): Raises NotFound if computeService does not exist. """ + return IMPL.compute_service_update(context, compute_id, values) @@ -446,27 +447,22 @@ def instance_add_security_group(context, instance_id, security_group_id): security_group_id) -def instance_get_all_by_host(context, hostname): - """Get instances by host""" - return IMPL.instance_get_all_by_host(context, hostname) - - def instance_get_vcpu_sum_by_host_and_project(context, hostname, proj_id): - """Get instances.vcpus by host and project""" + """Get instances.vcpus by host and project.""" return IMPL.instance_get_vcpu_sum_by_host_and_project(context, hostname, proj_id) def instance_get_memory_sum_by_host_and_project(context, hostname, proj_id): - """Get amount of memory by host and project """ + """Get amount of memory by host and project.""" return IMPL.instance_get_memory_sum_by_host_and_project(context, hostname, proj_id) def instance_get_disk_sum_by_host_and_project(context, hostname, proj_id): - """Get total amount of disk by host and project """ + """Get total amount of disk by host and project.""" return IMPL.instance_get_disk_sum_by_host_and_project(context, hostname, proj_id) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 43d56cd8a..b4f45a089 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -184,8 +184,8 @@ def service_get_all_compute_by_host(context, host): all() if not result: - msg = _('%s does not exist or not compute node') - raise exception.NotFound(msg % host) + raise exception.NotFound(_("%s does not exist or not " + "compute node.") % host) return result @@ -328,7 +328,7 @@ def compute_service_create(context, values): def compute_service_update(context, compute_id, values): session = get_session() with session.begin(): - compute_ref = service_get(context, compute_id, session=session) + compute_ref = compute_service_get(context, compute_id, session=session) compute_ref.update(values) compute_ref.save(session=session) @@ -964,21 +964,6 @@ def instance_add_security_group(context, instance_id, security_group_id): instance_ref.save(session=session) -@require_context -def instance_get_all_by_host(context, hostname): - session = get_session() - if not session: - session = get_session() - - result = session.query(models.Instance).\ - filter_by(host=hostname).\ - filter_by(deleted=can_read_deleted(context)).\ - all() - if not result: - return [] - return result - - @require_context def instance_get_vcpu_sum_by_host_and_project(context, hostname, proj_id): session = get_session() @@ -987,7 +972,7 @@ def instance_get_vcpu_sum_by_host_and_project(context, hostname, proj_id): filter_by(project_id=proj_id).\ filter_by(deleted=False).\ value(func.sum(models.Instance.vcpus)) - if None == result: + if not result: return 0 return result @@ -1000,7 +985,7 @@ def instance_get_memory_sum_by_host_and_project(context, hostname, proj_id): filter_by(project_id=proj_id).\ filter_by(deleted=False).\ value(func.sum(models.Instance.memory_mb)) - if None == result: + if not result: return 0 return result @@ -1013,7 +998,7 @@ def instance_get_disk_sum_by_host_and_project(context, hostname, proj_id): filter_by(project_id=proj_id).\ filter_by(deleted=False).\ value(func.sum(models.Instance.local_gb)) - if None == result: + if not result: return 0 return result diff --git a/nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py b/nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py index 903f7a646..2689b5b74 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py @@ -16,10 +16,9 @@ # License for the specific language governing permissions and limitations # under the License. -from sqlalchemy import * from migrate import * - from nova import log as logging +from sqlalchemy import * meta = MetaData() diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 02d4e2f9b..f2a029c20 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -137,12 +137,14 @@ class ComputeService(BASE, NovaBase): # Note(masumotok): Expected Strings example: # - # '{"arch":"x86_64", "model":"Nehalem", - # "topology":{"sockets":1, "threads":2, "cores":3}, - # features:[ "tdtscp", "xtpr"]}' + # '{"arch":"x86_64", + # "model":"Nehalem", + # "topology":{"sockets":1, "threads":2, "cores":3}, + # "features":["tdtscp", "xtpr"]}' # # Points are "json translatable" and it must have all dictionary keys - # above, and tag of getCapabilities()(See libvirt.virtConnection). + # above, since it is copied from tag of getCapabilities() + # (See libvirt.virtConnection). cpu_info = Column(Text, nullable=True) @@ -220,7 +222,7 @@ class Instance(BASE, NovaBase): display_description = Column(String(255)) # To remember on which host a instance booted. - # An instance may moved to other host by live migraiton. + # An instance may have moved to another host by live migraiton. launched_on = Column(Text) locked = Column(Boolean) diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 41b87bbca..8c30702ba 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -70,9 +70,18 @@ class Scheduler(object): raise NotImplementedError(_("Must implement a fallback schedule")) def schedule_live_migration(self, context, instance_id, dest): - """live migration method""" + """Live migration scheduling method. - # Whether instance exists and running + :param context: + :param instance_id: + :param dest: destination host + :return: + The host where instance is running currently. + Then scheduler send request that host. + + """ + + # Whether instance exists and is running. instance_ref = db.instance_get(context, instance_id) # Checking instance. @@ -102,11 +111,16 @@ class Scheduler(object): return src def _live_migration_src_check(self, context, instance_ref): - """Live migration check routine (for src host)""" + """Live migration check routine (for src host). + + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + + """ # Checking instance is running. - if power_state.RUNNING != instance_ref['state'] or \ - 'running' != instance_ref['state_description']: + if (power_state.RUNNING != instance_ref['state'] or \ + 'running' != instance_ref['state_description']): msg = _('Instance(%s) is not running') ec2_id = instance_ref['hostname'] raise exception.Invalid(msg % ec2_id) @@ -129,7 +143,13 @@ class Scheduler(object): raise exception.Invalid(msg % src) def _live_migration_dest_check(self, context, instance_ref, dest): - """Live migration check routine (for destination host)""" + """Live migration check routine (for destination host). + + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + :param dest: destination host + + """ # Checking dest exists and compute node. dservice_refs = db.service_get_all_compute_by_host(context, dest) @@ -145,20 +165,25 @@ class Scheduler(object): src = instance_ref['host'] if dest == src: ec2_id = instance_ref['hostname'] - msg = _("""%(dest)s is where %(ec2_id)s is """ - """running now. choose other host.""") % locals() - raise exception.Invalid(msg) + raise exception.Invalid(_("%(dest)s is where %(ec2_id)s is " + "running now. choose other host.") + % locals()) # Checking dst host still has enough capacities. - self.has_enough_resource(context, instance_ref, dest) + self.has_enough_resources(context, instance_ref, dest) def _live_migration_common_check(self, context, instance_ref, dest): - """ - Live migration check routine. - Below pre-checkings are followed by + """Live migration common check routine. + + Below checkings are followed by http://wiki.libvirt.org/page/TodoPreMigrationChecks + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + :param dest: destination host + """ + # Checking shared storage connectivity self.mounted_on_same_shared_storage(context, instance_ref, dest) @@ -168,27 +193,27 @@ class Scheduler(object): # Checking original host( where instance was launched at) exists. try: - oservice_refs = \ - db.service_get_all_compute_by_host(context, - instance_ref['launched_on']) + oservice_refs = db.service_get_all_compute_by_host(context, + instance_ref['launched_on']) except exception.NotFound: - msg = _('%s(where instance was launched at) does not exists.') - raise exception.Invalid(msg % instance_ref['launched_on']) + raise exception.Invalid(_("host %s where instance was launched " + "does not exist.") + % instance_ref['launched_on']) oservice_ref = oservice_refs[0]['compute_service'][0] # Checking hypervisor is same. o = oservice_ref['hypervisor_type'] d = dservice_ref['hypervisor_type'] if o != d: - msg = _('Different hypervisor type(%(o)s->%(d)s)') % locals() - raise exception.Invalid(msg) + raise exception.Invalid(_("Different hypervisor type" + "(%(o)s->%(d)s)')" % locals())) # Checkng hypervisor version. o = oservice_ref['hypervisor_version'] d = dservice_ref['hypervisor_version'] if o > d: - msg = _('Older hypervisor version(%(o)s->%(d)s)') % locals() - raise exception.Invalid(msg) + raise exception.Invalid(_('Older hypervisor version(%(o)s->%(d)s)') + % locals()) # Checking cpuinfo. try: @@ -200,19 +225,24 @@ class Scheduler(object): except rpc.RemoteError, e: ec2_id = instance_ref['hostname'] src = instance_ref['host'] - msg = _("""%(dest)s doesnt have compatibility to %(src)s""" - """(where %(ec2_id)s was launched at)""") - logging.exception(msg % locals()) - raise e + logging.exception(_("host %(dest)s is not compatible with " + "original host %(src)s.") % locals()) + raise + + def has_enough_resources(self, context, instance_ref, dest): + """Checks if destination host has enough resource for live migration. - def has_enough_resource(self, context, instance_ref, dest): - """ - Check if destination host has enough resource for live migration. Currently, only memory checking has been done. If storage migration(block migration, meaning live-migration without any shared storage) will be available, local storage checking is also necessary. + + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + :param dest: destination host + """ + # Getting instance information ec2_id = instance_ref['hostname'] @@ -225,15 +255,23 @@ class Scheduler(object): mem_avail = mem_total - mem_used mem_inst = instance_ref['memory_mb'] if mem_avail <= mem_inst: - msg = _("""%(ec2_id)s is not capable to migrate %(dest)s""" - """(host:%(mem_avail)s <= instance:%(mem_inst)s)""") - raise exception.NotEmpty(msg % locals()) + raise exception.NotEmpty(_("%(ec2_id)s is not capable to " + "migrate %(dest)s (host:%(mem_avail)s " + " <= instance:%(mem_inst)s)") + % locals()) def mounted_on_same_shared_storage(self, context, instance_ref, dest): + """Check if the src and dest host mount same shared storage. + + At first, dest host creates temp file, and src host can see + it if they mounts same shared storage. Then src host erase it. + + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + :param dest: destination host + """ - Check if /nova-inst-dir/insntances is mounted same storage at - live-migration src and dest host. - """ + src = instance_ref['host'] dst_t = db.queue_get_for(context, FLAGS.compute_topic, dest) src_t = db.queue_get_for(context, FLAGS.compute_topic, src) @@ -243,8 +281,8 @@ class Scheduler(object): filename = rpc.call(context, dst_t, {"method": 'mktmpfile'}) except rpc.RemoteError, e: msg = _("Cannot create tmpfile at %s to confirm shared storage.") - logging.error(msg % FLAGS.instance_path) - raise e + LOG.error(msg % FLAGS.instances_path) + raise # make sure existence at src host. try: @@ -252,8 +290,7 @@ class Scheduler(object): {"method": 'confirm_tmpfile', "args": {'path': filename}}) except (rpc.RemoteError, exception.NotFound), e: - ipath = FLAGS.instance_path - msg = _("""Cannot comfirm %(ipath)s at %(dest)s is located at""" - """ same shared storage.""") % locals() - logging.error(msg) - raise e + ipath = FLAGS.instances_path + logging.error(_("Cannot comfirm %(ipath)s at %(dest)s is " + "located at same shared storage.") % locals()) + raise diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 584dc49d2..783594c6f 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -69,10 +69,19 @@ class SchedulerManager(manager.Manager): LOG.debug(_("Casting to %(topic)s %(host)s for %(method)s") % locals()) # NOTE (masumotok) : This method should be moved to nova.api.ec2.admin. - # Based on bear design summit discussion, + # Based on bexar design summit discussion, # just put this here for bexar release. - def show_host_resource(self, context, host, *args): - """show the physical/usage resource given by hosts.""" + def show_host_resources(self, context, host, *args): + """Shows the physical/usage resource given by hosts. + + :param context: security context + :param host: hostname + :returns: + example format is below. + {'resource':D, 'usage':{proj_id1:D, proj_id2:D}} + D: {'vcpus':3, 'memory_mb':2048, 'local_gb':2048} + + """ compute_ref = db.service_get_all_compute_by_host(context, host) compute_ref = compute_ref[0] diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 74cb82eeb..3c88d186d 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -314,10 +314,7 @@ class ComputeTestCase(test.TestCase): self.compute_driver = utils.import_object(FLAGS.compute_driver) def test_pre_live_migration_instance_has_no_fixed_ip(self): - """ - if instances that are intended to be migrated doesnt have fixed_ip - (not happens usually), pre_live_migration has to raise Exception. - """ + """Confirm raising exception if instance doesn't have fixed_ip.""" instance_ref = self._get_dummy_instance() c = context.get_admin_context() i_id = instance_ref['id'] @@ -331,14 +328,9 @@ class ComputeTestCase(test.TestCase): self.assertRaises(exception.NotFound, self.compute.pre_live_migration, c, instance_ref['id']) - self.mox.ResetAll() def test_pre_live_migration_instance_has_volume(self): - """if any volumes are attached to the instances that are - intended to be migrated, setup_compute_volume must be - called because aoe module should be inserted at destination - host. This testcase checks on it. - """ + """Confirm setup_compute_volume is called when volume is mounted.""" i_ref = self._get_dummy_instance() c = context.get_admin_context() @@ -364,14 +356,9 @@ class ComputeTestCase(test.TestCase): self.mox.ReplayAll() ret = self.compute.pre_live_migration(c, i_ref['id']) self.assertEqual(ret, None) - self.mox.ResetAll() def test_pre_live_migration_instance_has_no_volume(self): - """if any volumes are not attached to the instances that are - intended to be migrated, log message should be appears - because administrator can proove instance conditions before - live_migration if any trouble occurs. - """ + """Confirm log meg when instance doesn't mount any volumes.""" i_ref = self._get_dummy_instance() i_ref.__setitem__('volumes', []) c = context.get_admin_context() @@ -395,14 +382,14 @@ class ComputeTestCase(test.TestCase): self.mox.ReplayAll() ret = self.compute.pre_live_migration(c, i_ref['id']) self.assertEqual(ret, None) - self.mox.ResetAll() def test_pre_live_migration_setup_compute_node_fail(self): - """setup_compute_node sometimes fail since concurrent request - comes to iptables and iptables complains. Then this method - tries to retry, but raise exception in case of over - max_retry_count. this method confirms raising exception. + """Confirm operation setup_compute_network() fails. + + It retries and raise exception when timeout exceeded. + """ + i_ref = self._get_dummy_instance() c = context.get_admin_context() @@ -427,14 +414,9 @@ class ComputeTestCase(test.TestCase): self.assertRaises(exception.ProcessExecutionError, self.compute.pre_live_migration, c, i_ref['id']) - self.mox.ResetAll() - def test_live_migration_instance_has_volume(self): - """Any volumes are mounted by instances to be migrated are found, - vblade health must be checked before starting live-migration. - And that is checked by check_for_export(). - This testcase confirms check_for_export() is called. - """ + def test_live_migration_works_correctly_with_volume(self): + """Confirm check_for_export to confirm volume health check.""" i_ref = self._get_dummy_instance() c = context.get_admin_context() topic = db.queue_get_for(c, FLAGS.compute_topic, i_ref['host']) @@ -457,15 +439,9 @@ class ComputeTestCase(test.TestCase): self.mox.ReplayAll() ret = self.compute.live_migration(c, i_ref['id'], i_ref['host']) self.assertEqual(ret, None) - self.mox.ResetAll() - - def test_live_migration_instance_has_volume_and_exception(self): - """In addition to test_live_migration_instance_has_volume testcase, - this testcase confirms if any exception raises from - check_for_export(). Then, valid seaquence of this method should - recovering instance/volumes status(ex. instance['state_description'] - is changed from 'migrating' -> 'running', was changed by scheduler) - """ + + def test_live_migration_dest_raises_exception(self): + """Confirm exception when pre_live_migration fails.""" i_ref = self._get_dummy_instance() c = context.get_admin_context() topic = db.queue_get_for(c, FLAGS.compute_topic, i_ref['host']) @@ -484,20 +460,16 @@ class ComputeTestCase(test.TestCase): 'state': power_state.RUNNING, 'host': i_ref['host']}) for v in i_ref['volumes']: - dbmock.volume_update(c, v['id'], {'status': 'in-use', - 'host': i_ref['host']}) + dbmock.volume_update(c, v['id'], {'status': 'in-use'}) self.compute.db = dbmock self.mox.ReplayAll() self.assertRaises(rpc.RemoteError, self.compute.live_migration, c, i_ref['id'], i_ref['host']) - self.mox.ResetAll() - def test_live_migration_instance_has_no_volume_and_exception(self): - """Simpler than - test_live_migration_instance_has_volume_and_exception - """ + def test_live_migration_dest_raises_exception_no_volume(self): + """Same as above test(input pattern is different) """ i_ref = self._get_dummy_instance() i_ref.__setitem__('volumes', []) c = context.get_admin_context() @@ -520,10 +492,9 @@ class ComputeTestCase(test.TestCase): self.assertRaises(rpc.RemoteError, self.compute.live_migration, c, i_ref['id'], i_ref['host']) - self.mox.ResetAll() - def test_live_migration_instance_has_no_volume(self): - """Simpler than test_live_migration_instance_has_volume.""" + def test_live_migration_works_correctly_no_volume(self): + """Confirm live_migration() works as expected correctly.""" i_ref = self._get_dummy_instance() i_ref.__setitem__('volumes', []) c = context.get_admin_context() @@ -545,11 +516,9 @@ class ComputeTestCase(test.TestCase): self.mox.ReplayAll() ret = self.compute.live_migration(c, i_ref['id'], i_ref['host']) self.assertEqual(ret, None) - self.mox.ResetAll() def test_post_live_migration_working_correctly(self): - """post_live_migration works as expected correctly """ - + """Confirm post_live_migration() works as expected correctly.""" dest = 'desthost' flo_addr = '1.2.1.2' @@ -579,19 +548,15 @@ class ComputeTestCase(test.TestCase): # executing self.mox.ReplayAll() ret = self.compute.post_live_migration(c, i_ref, dest) - self.mox.UnsetStubs() # make sure every data is rewritten to dest i_ref = db.instance_get(c, i_ref['id']) c1 = (i_ref['host'] == dest) - v_ref = db.volume_get(c, v_ref['id']) - c2 = (v_ref['host'] == dest) - c3 = False flo_refs = db.floating_ip_get_all_by_host(c, dest) - c3 = (len(flo_refs) != 0 and flo_refs[0]['address'] == flo_addr) + c2 = (len(flo_refs) != 0 and flo_refs[0]['address'] == flo_addr) # post operaton - self.assertTrue(c1 and c2 and c3) + self.assertTrue(c1 and c2) db.instance_destroy(c, instance_id) db.volume_destroy(c, v_ref['id']) db.floating_ip_destroy(c, flo_addr) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 729bcb580..301106848 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -108,22 +108,21 @@ class SchedulerTestCase(test.TestCase): self.mox.ReplayAll() scheduler.named_method(ctxt, 'topic', num=7) - def test_show_host_resource_host_not_exit(self): - """ - A testcase of driver.has_enough_resource - given host does not exists. - """ + def test_show_host_resources_host_not_exit(self): + """A host given as an argument does not exists.""" + scheduler = manager.SchedulerManager() dest = 'dummydest' ctxt = context.get_admin_context() try: - scheduler.show_host_resource(ctxt, dest) + scheduler.show_host_resources(ctxt, dest) except exception.NotFound, e: c1 = (0 <= e.message.find('does not exist or not compute node')) self.assertTrue(c1) def _dic_is_equal(self, dic1, dic2, keys=None): + """Compares 2 dictionary contents(Helper method)""" if not keys: keys = ['vcpus', 'memory_mb', 'local_gb', 'vcpus_used', 'memory_mb_used', 'local_gb_used'] @@ -133,16 +132,14 @@ class SchedulerTestCase(test.TestCase): return False return True - def test_show_host_resource_no_project(self): - """ - A testcase of driver.show_host_resource - no instance stays on the given host - """ + def test_show_host_resources_no_project(self): + """No instance are running on the given host.""" + scheduler = manager.SchedulerManager() ctxt = context.get_admin_context() s_ref = self._create_compute_service() - result = scheduler.show_host_resource(ctxt, s_ref['host']) + result = scheduler.show_host_resources(ctxt, s_ref['host']) # result checking c1 = ('resource' in result and 'usage' in result) @@ -152,11 +149,9 @@ class SchedulerTestCase(test.TestCase): self.assertTrue(c1 and c2 and c3) db.service_destroy(ctxt, s_ref['id']) - def test_show_host_resource_works_correctly(self): - """ - A testcase of driver.show_host_resource - to make sure everything finished with no error. - """ + def test_show_host_resources_works_correctly(self): + """show_host_resources() works correctly as expected.""" + scheduler = manager.SchedulerManager() ctxt = context.get_admin_context() s_ref = self._create_compute_service() @@ -164,7 +159,7 @@ class SchedulerTestCase(test.TestCase): i_ref2 = self._create_instance(project_id='p-02', vcpus=3, host=s_ref['host']) - result = scheduler.show_host_resource(ctxt, s_ref['host']) + result = scheduler.show_host_resources(ctxt, s_ref['host']) c1 = ('resource' in result and 'usage' in result) compute_service = s_ref['compute_service'][0] @@ -284,6 +279,7 @@ class SimpleDriverTestCase(test.TestCase): return db.volume_create(self.context, vol)['id'] def _create_compute_service(self, **kwargs): + """Create a compute service.""" dic = {'binary': 'nova-compute', 'topic': 'compute', 'report_count': 0, 'availability_zone': 'dummyzone'} @@ -698,13 +694,13 @@ class SimpleDriverTestCase(test.TestCase): volume1.kill() volume2.kill() - def test_scheduler_live_migraiton_with_volume(self): - """ - driver.scheduler_live_migration finishes successfully - (volumes are attached to instances) - This testcase make sure schedule_live_migration - changes instance state from 'running' -> 'migrating' + def test_scheduler_live_migration_with_volume(self): + """scheduler_live_migration() works correctly as expected. + + Also, checks instance state is changed from 'running' -> 'migrating'. + """ + instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) dic = {'instance_id': instance_id, 'size': 1} @@ -737,11 +733,9 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.volume_destroy(self.context, v_ref['id']) - def test_live_migraiton_src_check_instance_not_running(self): - """ - A testcase of driver._live_migration_src_check. - The instance given by instance_id is not running. - """ + def test_live_migration_src_check_instance_not_running(self): + """The instance given by instance_id is not running.""" + instance_id = self._create_instance(state_description='migrating') i_ref = db.instance_get(self.context, instance_id) @@ -754,12 +748,9 @@ class SimpleDriverTestCase(test.TestCase): self.assertTrue(c) db.instance_destroy(self.context, instance_id) - def test_live_migraiton_src_check_volume_node_not_alive(self): - """ - A testcase of driver._live_migration_src_check. - Volume node is not alive if any volumes are attached to - the given instance. - """ + def test_live_migration_src_check_volume_node_not_alive(self): + """Raise exception when volume node is not alive.""" + instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) dic = {'instance_id': instance_id, 'size': 1} @@ -782,11 +773,8 @@ class SimpleDriverTestCase(test.TestCase): db.service_destroy(self.context, s_ref['id']) db.volume_destroy(self.context, v_ref['id']) - def test_live_migraiton_src_check_compute_node_not_alive(self): - """ - A testcase of driver._live_migration_src_check. - The testcase make sure src-compute node is alive. - """ + def test_live_migration_src_check_compute_node_not_alive(self): + """Confirms src-compute node is alive.""" instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) t = datetime.datetime.utcnow() - datetime.timedelta(10) @@ -803,11 +791,8 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) - def test_live_migraiton_src_check_works_correctly(self): - """ - A testcase of driver._live_migration_src_check. - The testcase make sure everything finished with no error. - """ + def test_live_migration_src_check_works_correctly(self): + """Confirms this method finishes with no error.""" instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) s_ref = self._create_compute_service(host=i_ref['host']) @@ -819,11 +804,8 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) - def test_live_migraiton_dest_check_not_alive(self): - """ - A testcase of driver._live_migration_dst_check. - Destination host does not exist. - """ + def test_live_migration_dest_check_not_alive(self): + """Confirms exception raises in case dest host does not exist.""" instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) t = datetime.datetime.utcnow() - datetime.timedelta(10) @@ -841,11 +823,8 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) - def test_live_migraiton_dest_check_service_same_host(self): - """ - A testcase of driver._live_migration_dst_check. - Destination host is same as src host. - """ + def test_live_migration_dest_check_service_same_host(self): + """Confirms exceptioin raises in case dest and src is same host.""" instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) s_ref = self._create_compute_service(host=i_ref['host']) @@ -861,11 +840,8 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) - def test_live_migraiton_dest_check_service_lack_memory(self): - """ - A testcase of driver._live_migration_dst_check. - destination host doesnt have enough memory. - """ + def test_live_migration_dest_check_service_lack_memory(self): + """Confirms exception raises when dest doesn't have enough memory.""" instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) s_ref = self._create_compute_service(host='somewhere', @@ -882,11 +858,8 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) - def test_live_migraiton_dest_check_service_works_correctly(self): - """ - A testcase of driver._live_migration_dst_check. - The testcase make sure everything finished with no error. - """ + def test_live_migration_dest_check_service_works_correctly(self): + """Confirms method finishes with no error.""" instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) s_ref = self._create_compute_service(host='somewhere', @@ -899,13 +872,11 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) - def test_live_migraiton_common_check_service_orig_not_exists(self): - """ - A testcase of driver._live_migration_common_check. - Destination host does not exist. - """ + def test_live_migration_common_check_service_orig_not_exists(self): + """Destination host does not exist.""" + dest = 'dummydest' - # mocks for live_migraiton_common_check() + # mocks for live_migration_common_check() instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) t1 = datetime.datetime.utcnow() - datetime.timedelta(10) @@ -929,18 +900,15 @@ class SimpleDriverTestCase(test.TestCase): i_ref, dest) except exception.Invalid, e: - c = (e.message.find('does not exists') >= 0) + c = (e.message.find('does not exist') >= 0) self.assertTrue(c) self.mox.UnsetStubs() db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) - def test_live_migraiton_common_check_service_different_hypervisor(self): - """ - A testcase of driver._live_migration_common_check. - Original host and dest host has different hypervisor type. - """ + def test_live_migration_common_check_service_different_hypervisor(self): + """Original host and dest host has different hypervisor type.""" dest = 'dummydest' instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) @@ -969,11 +937,8 @@ class SimpleDriverTestCase(test.TestCase): db.service_destroy(self.context, s_ref['id']) db.service_destroy(self.context, s_ref2['id']) - def test_live_migraiton_common_check_service_different_version(self): - """ - A testcase of driver._live_migration_common_check. - Original host and dest host has different hypervisor version. - """ + def test_live_migration_common_check_service_different_version(self): + """Original host and dest host has different hypervisor version.""" dest = 'dummydest' instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) @@ -1003,11 +968,9 @@ class SimpleDriverTestCase(test.TestCase): db.service_destroy(self.context, s_ref['id']) db.service_destroy(self.context, s_ref2['id']) - def test_live_migraiton_common_check_service_checking_cpuinfo_fail(self): - """ - A testcase of driver._live_migration_common_check. - Original host and dest host has different hypervisor version. - """ + def test_live_migration_common_check_checking_cpuinfo_fail(self): + """Raise excetion when original host doen't have compatible cpu.""" + dest = 'dummydest' instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) @@ -1025,7 +988,7 @@ class SimpleDriverTestCase(test.TestCase): rpc.call(mox.IgnoreArg(), mox.IgnoreArg(), {"method": 'compare_cpu', "args": {'cpu_info': s_ref2['compute_service'][0]['cpu_info']}}).\ - AndRaise(rpc.RemoteError('doesnt have compatibility to', '', '')) + AndRaise(rpc.RemoteError("doesn't have compatibility to", "", "")) self.mox.ReplayAll() try: @@ -1033,7 +996,7 @@ class SimpleDriverTestCase(test.TestCase): i_ref, dest) except rpc.RemoteError, e: - c = (e.message.find(_('doesnt have compatibility to')) >= 0) + c = (e.message.find(_("doesn't have compatibility to")) >= 0) self.assertTrue(c) self.mox.UnsetStubs() diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index cb65584cf..bbd5c6d92 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -279,11 +279,7 @@ class ServiceTestCase(test.TestCase): self.assert_(not serv.model_disconnected) def test_compute_can_update_available_resource(self): - """ - Test nova-compute successfully updated Service table on DB. - Doing so, self.manager.update_service must be called - if 'self.binary == nova-compute', and this testcase checks on it. - """ + """Confirm compute updates their record of compute-service table.""" host = 'foo' binary = 'nova-compute' topic = 'compute1' diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 8ed726c21..91bdfcc5a 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -23,8 +23,8 @@ from nova import context from nova import db from nova import exception from nova import flags -from nova import test from nova import logging +from nova import test from nova import utils from nova.api.ec2 import cloud from nova.auth import manager @@ -76,12 +76,12 @@ class LibvirtConnTestCase(test.TestCase): 'bridge': 'br101', 'instance_type': 'm1.small'} - def _driver_dependent_test_setup(self): - """ - Setup method. - Call this method at the top of each testcase method, - if the testcase is necessary libvirt and cheetah. - """ + def _driver_dependant_test_setup(self): + """Call this method at the top of each testcase method. + + Checks if libvirt and cheetah, etc is installed. + Otherwise, skip testing.""" + try: global libvirt global libxml2 @@ -92,10 +92,9 @@ class LibvirtConnTestCase(test.TestCase): except ImportError, e: logging.warn("""This test has not been done since """ """using driver-dependent library Cheetah/libvirt/libxml2.""") - raise e + raise # inebitable mocks for calling - #nova.virt.libvirt_conn.LibvirtConnection.__init__ obj = utils.import_object(FLAGS.firewall_driver) fwmock = self.mox.CreateMock(obj) self.mox.StubOutWithMock(libvirt_conn, 'utils', @@ -258,51 +257,31 @@ class LibvirtConnTestCase(test.TestCase): self.assertEquals(uri, testuri) def test_get_vcpu_total(self): - """ - Check if get_vcpu_total returns appropriate cpu value - Connection/OS/driver differenct does not matter for this method, - everyone can execute for checking. - """ + """Check if get_vcpu_total returns appropriate cpu value.""" try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertTrue(0 < conn.get_vcpu_total()) - self.mox.UnsetStubs() def test_get_memory_mb_total(self): - """Check if get_memory_mb returns appropriate memory value""" + """Check if get_memory_mb returns appropriate memory value.""" try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertTrue(0 < conn.get_memory_mb_total()) - self.mox.UnsetStubs() - - def test_get_local_gb_total(self): - """Check if get_local_gb_total returns appropriate disk value""" - # Note(masumotok): leave this b/c FLAGS.instances_path is inevitable.. - #try: - # self._driver_dependent_test_setup() - #except: - # return - # - #self.mox.ReplayAll() - #conn = libvirt_conn.LibvirtConnection(False) - #self.assertTrue(0 < conn.get_local_gb_total()) - #self.mox.UnsetStubs() - pass def test_get_vcpu_used(self): - """Check if get_local_gb_total returns appropriate disk value""" + """Check if get_local_gb_total returns appropriate disk value.""" try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return @@ -321,52 +300,45 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertTrue(conn.get_vcpu_used() == 4) - self.mox.UnsetStubs() def test_get_memory_mb_used(self): - """Check if get_memory_mb returns appropriate memory value""" + """Check if get_memory_mb returns appropriate memory value.""" try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertTrue(0 < conn.get_memory_mb_used()) - self.mox.UnsetStubs() - - def test_get_local_gb_used(self): - """Check if get_local_gb_total returns appropriate disk value""" - # Note(masumotok): leave this b/c FLAGS.instances_path is inevitable - #try: - # self._driver_dependent_test_setup() - #except: - # return - - #self.mox.ReplayAll() - #conn = libvirt_conn.LibvirtConnection(False) - #self.assertTrue(0 < conn.get_local_gb_used()) - #self.mox.UnsetStubs() - pass def test_get_cpu_info_works_correctly(self): - """ - Check if get_cpu_info works correctly. - (in case libvirt.getCapabilities() works correctly) - """ - xml = ("""x86_64Nehalem""" - """Intel""" - """""" - """""" - """""" - """""" - """""" - """""" - """""") + """Check if get_cpu_info works correctly as expected.""" + xml = """ + x86_64 + Nehalem + Intel + + + + + + + + + + + + + + + + + + """ try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, @@ -376,27 +348,34 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertTrue(0 < len(conn.get_cpu_info())) - self.mox.UnsetStubs() def test_get_cpu_info_inappropreate_xml(self): - """ - Check if get_cpu_info raises exception - in case libvirt.getCapabilities() returns wrong xml - (in case of xml doesnt have tag) - """ - xml = ("""x86_64Nehalem""" - """Intel""" - """""" - """""" - """""" - """""" - """""" - """""" - """""") + """Raise exception if given xml is inappropriate.""" + xml = """ + x86_64 + Nehalem + Intel + + + + + + + + + + + + + + + + + + """ try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, @@ -409,29 +388,34 @@ class LibvirtConnTestCase(test.TestCase): conn.get_cpu_info() except exception.Invalid, e: c1 = (0 <= e.message.find('Invalid xml')) - self.assertTrue(c1) - self.mox.UnsetStubs() + self.assertTrue(c1) def test_get_cpu_info_inappropreate_xml2(self): - """ - Check if get_cpu_info raises exception - in case libvirt.getCapabilities() returns wrong xml - (in case of xml doesnt have inproper tag - meaning missing "socket" attribute) - """ - xml = ("""x86_64Nehalem""" - """Intel""" - """""" - """""" - """""" - """""" - """""" - """""" - """""") - + """Raise exception if given xml is inappropriate(topology tag).""" + + xml = """ + x86_64 + Nehalem + Intel + + + + + + + + + + + + + + + + + """ try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, @@ -444,29 +428,12 @@ class LibvirtConnTestCase(test.TestCase): conn.get_cpu_info() except exception.Invalid, e: c1 = (0 <= e.message.find('Invalid xml: topology')) - self.assertTrue(c1) - self.mox.UnsetStubs() + self.assertTrue(c1) def test_update_available_resource_works_correctly(self): - """ - In this method, vcpus/memory_mb/local_gb/vcpu_used/ - memory_mb_used/local_gb_used/hypervisor_type/ - hypervisor_version/cpu_info should be changed. - Based on this specification, this testcase confirms - if this method finishes successfully, - meaning self.db.service_update must be called with dictinary - - {'vcpu':aaa, 'memory_mb':bbb, 'local_gb':ccc, - 'vcpu_used':aaa, 'memory_mb_used':bbb, 'local_gb_sed':ccc, - 'hypervisor_type':ddd, 'hypervisor_version':eee, - 'cpu_info':fff} - - Since each value of above dict can be obtained through - driver(different depends on environment), - only dictionary keys are checked. - """ + """Confirm compute_service table is updated successfully.""" try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return @@ -478,7 +445,9 @@ class LibvirtConnTestCase(test.TestCase): host = 'foo' binary = 'nova-compute' - service_ref = {'id': 1, 'host': host, 'binary': binary, + service_ref = {'id': 1, + 'host': host, + 'binary': binary, 'topic': 'compute'} self.mox.StubOutWithMock(db, 'service_get_all_by_topic') @@ -491,15 +460,11 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) conn.update_available_resource(host) - self.mox.UnsetStubs() def test_update_resource_info_raise_exception(self): - """ - This testcase confirms if no record found on Service - table, exception can be raised. - """ + """Raise exception if no recorde found on services table.""" try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return @@ -518,18 +483,19 @@ class LibvirtConnTestCase(test.TestCase): msg = 'Cannot insert compute manager specific info' c1 = (0 <= e.message.find(msg)) self.assertTrue(c1) - self.mox.ResetAll() def test_compare_cpu_works_correctly(self): - """Calling libvirt.compute_cpu() and works correctly """ - - t = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ - """"topology":{"cores":"%s", "threads":"%s", """ - """"sockets":"%s"}, "features":[%s]}""") - cpu_info = t % ('x86', 'model', 'vendor', '2', '1', '4', '"tm"') + """Calling libvirt.compute_cpu() and works correctly.""" + t = {} + t['arch'] = 'x86' + t['model'] = 'model' + t['vendor'] = 'Intel' + t['topology'] = {'cores': "2", "threads": "1", "sockets": "4"} + t['features'] = ["tm"] + cpu_info = utils.dumps(t) try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return @@ -542,20 +508,19 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertTrue(None == conn.compare_cpu(cpu_info)) - self.mox.UnsetStubs() def test_compare_cpu_raises_exception(self): - """ - Libvirt-related exception occurs when calling - libvirt.compare_cpu(). - """ - t = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ - """"topology":{"cores":"%s", "threads":"%s", """ - """"sockets":"%s"}, "features":[%s]}""") - cpu_info = t % ('x86', 'model', 'vendor', '2', '1', '4', '"tm"') + """Libvirt-related exception occurs when calling compare_cpu().""" + t = {} + t['arch'] = 'x86' + t['model'] = 'model' + t['vendor'] = 'Intel' + t['topology'] = {'cores': "2", "threads": "1", "sockets": "4"} + t['features'] = ["tm"] + cpu_info = utils.dumps(t) try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return @@ -567,18 +532,19 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertRaises(libvirt.libvirtError, conn.compare_cpu, cpu_info) - self.mox.UnsetStubs() def test_compare_cpu_no_compatibility(self): - """libvirt.compare_cpu() return less than 0.(no compatibility)""" - - t = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ - """"topology":{"cores":"%s", "threads":"%s", """ - """"sockets":"%s"}, "features":[%s]}""") - cpu_info = t % ('x86', 'model', 'vendor', '2', '1', '4', '"tm"') + """Libvirt.compare_cpu() return less than 0.(no compatibility).""" + t = {} + t['arch'] = 'x86' + t['model'] = 'model' + t['vendor'] = 'Intel' + t['topology'] = {'cores': "2", "threads": "1", "sockets": "4"} + t['features'] = ["tm"] + cpu_info = utils.dumps(t) try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return @@ -590,16 +556,14 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertRaises(exception.Invalid, conn.compare_cpu, cpu_info) - self.mox.UnsetStubs() def test_ensure_filtering_rules_for_instance_works_correctly(self): - """ensure_filtering_rules_for_instance works as expected correctly""" - + """ensure_filtering_rules_for_instance() works successfully.""" instance_ref = models.Instance() instance_ref.__setitem__('id', 1) try: - nwmock, fwmock = self._driver_dependent_test_setup() + nwmock, fwmock = self._driver_dependant_test_setup() except: return @@ -613,16 +577,14 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) conn.ensure_filtering_rules_for_instance(instance_ref) - self.mox.UnsetStubs() def test_ensure_filtering_rules_for_instance_timeout(self): - """ensure_filtering_fules_for_instance finishes with timeout""" - + """ensure_filtering_fules_for_instance() finishes with timeout.""" instance_ref = models.Instance() instance_ref.__setitem__('id', 1) try: - nwmock, fwmock = self._driver_dependent_test_setup() + nwmock, fwmock = self._driver_dependant_test_setup() except: return @@ -642,11 +604,9 @@ class LibvirtConnTestCase(test.TestCase): except exception.Error, e: c1 = (0 <= e.message.find('Timeout migrating for')) self.assertTrue(c1) - self.mox.UnsetStubs() def test_live_migration_works_correctly(self): - """_live_migration works as expected correctly """ - + """_live_migration() works as expected correctly.""" class dummyCall(object): f = None @@ -659,7 +619,7 @@ class LibvirtConnTestCase(test.TestCase): ctxt = context.get_admin_context() try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return @@ -681,13 +641,9 @@ class LibvirtConnTestCase(test.TestCase): # Not setting post_method/recover_method in this testcase. ret = conn._live_migration(ctxt, i_ref, i_ref['host'], '', '') self.assertTrue(ret == None) - self.mox.UnsetStubs() def test_live_migration_raises_exception(self): - """ - _live_migration raises exception, then this testcase confirms - recovered method is called. - """ + """Confirms recover method is called when exceptions are raised.""" i_ref = models.Instance() i_ref.__setitem__('id', 1) i_ref.__setitem__('host', 'dummy') @@ -697,7 +653,7 @@ class LibvirtConnTestCase(test.TestCase): pass try: - nwmock, fwmock = self._driver_dependent_test_setup() + nwmock, fwmock = self._driver_dependant_test_setup() except: return @@ -724,7 +680,6 @@ class LibvirtConnTestCase(test.TestCase): conn._mlive_migration, ctxt, instance_ref, dest, '', dummy_recover_method) - self.mox.UnsetStubs() def tearDown(self): super(LibvirtConnTestCase, self).tearDown() diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 70ddd3aaf..069a424d1 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -326,59 +326,6 @@ class FakeConnection(object): 'username': 'fakeuser', 'password': 'fakepassword'} - def get_cpu_info(self): - """This method is supported only libvirt. """ - return - - def get_vcpu_total(self): - """This method is supported only libvirt. """ - return - - def get_memory_mb_total(self): - """This method is supported only libvirt. """ - return - - def get_local_gb_total(self): - """This method is supported only libvirt. """ - return - - def get_vcpu_used(self): - """This method is supported only libvirt. """ - return - - def get_memory_mb_used(self): - """This method is supported only libvirt. """ - return - - def get_local_gb_used(self): - """This method is supported only libvirt. """ - return - - def get_hypervisor_type(self): - """This method is supported only libvirt..""" - return - - def get_hypervisor_version(self): - """This method is supported only libvirt..""" - return - - def compare_cpu(self, xml): - """This method is supported only libvirt..""" - raise NotImplementedError('This method is supported only libvirt.') - - def ensure_filtering_rules_for_instance(self, instance_ref): - """This method is supported only libvirt..""" - return - - def live_migration(self, context, instance_ref, dest, - post_method, recover_method): - """This method is supported only libvirt..""" - return - - def unfilter_instance(self, instance_ref): - """This method is supported only libvirt..""" - raise NotImplementedError('This method is supported only libvirt.') - def refresh_security_group_rules(self, security_group_id): """This method is called after a change to security groups. @@ -428,20 +375,25 @@ class FakeConnection(object): return True def update_available_resource(self, ctxt, host): - """This method is supported only libvirt. """ + """This method is supported only by libvirt.""" return def compare_cpu(self, xml): - """This method is supported only libvirt..""" - raise NotImplementedError('This method is supported only libvirt.') + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') def ensure_filtering_rules_for_instance(self, instance_ref): - """This method is supported only libvirt..""" - raise NotImplementedError('This method is supported only libvirt.') + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') + + def live_migration(self, context, instance_ref, dest, + post_method, recover_method): + """This method is supported only by libvirt.""" + return - def live_migration(self, context, instance_ref, dest): - """This method is supported only libvirt..""" - raise NotImplementedError('This method is supported only libvirt.') + def unfilter_instance(self, instance_ref): + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') class FakeInstance(object): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index d39836e72..934aed960 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -36,7 +36,6 @@ Supports KVM, QEMU, UML, and XEN. """ -import json import os import shutil import random @@ -105,7 +104,7 @@ flags.DEFINE_string('firewall_driver', 'Firewall driver (defaults to iptables)') flags.DEFINE_string('cpuinfo_xml_template', utils.abspath('virt/cpuinfo.xml.template'), - 'CpuInfo XML Template (used only live migration now)') + 'CpuInfo XML Template (Used only live migration now)') flags.DEFINE_string('live_migration_uri', "qemu+tcp://%s/system", 'Define protocol used by live_migration feature') @@ -851,23 +850,46 @@ class LibvirtConnection(object): return interfaces def get_vcpu_total(self): - """ Get vcpu number of physical computer. """ + """Get vcpu number of physical computer. + + :returns: the number of cpu core. + + """ + return open('/proc/cpuinfo').read().count('processor') def get_memory_mb_total(self): - """Get the total memory size(MB) of physical computer .""" + """Get the total memory size(MB) of physical computer. + + :returns: the total amount of memory(MB). + + """ + meminfo = open('/proc/meminfo').read().split() idx = meminfo.index('MemTotal:') # transforming kb to mb. return int(meminfo[idx + 1]) / 1024 def get_local_gb_total(self): - """Get the total hdd size(GB) of physical computer .""" + """Get the total hdd size(GB) of physical computer. + + :returns: + The total amount of HDD(GB). + Note that this value shows a partition where + NOVA-INST-DIR/instances mounts. + + """ + hddinfo = os.statvfs(FLAGS.instances_path) return hddinfo.f_frsize * hddinfo.f_blocks / 1024 / 1024 / 1024 def get_vcpu_used(self): - """ Get vcpu available number of physical computer. """ + """ Get vcpu usage number of physical computer. + + :returns: The total number of vcpu that currently used. + + """ + total = 0 for i in self._conn.listDomainsID(): dom = self._conn.lookupByID(i) @@ -875,7 +897,12 @@ class LibvirtConnection(object): return total def get_memory_mb_used(self): - """Get the free memory size(MB) of physical computer.""" + """Get the free memory size(MB) of physical computer. + + :returns: the total usage of memory(MB). + + """ + m = open('/proc/meminfo').read().split() idx1 = m.index('MemFree:') idx2 = m.index('Buffers:') @@ -884,21 +911,47 @@ class LibvirtConnection(object): return self.get_memory_mb_total() - avail def get_local_gb_used(self): - """Get the free hdd size(GB) of physical computer .""" + """Get the free hdd size(GB) of physical computer. + + :returns: + The total usage of HDD(GB). + Note that this value shows a partition where + NOVA-INST-DIR/instances mounts. + + """ + hddinfo = os.statvfs(FLAGS.instances_path) avail = hddinfo.f_frsize * hddinfo.f_bavail / 1024 / 1024 / 1024 return self.get_local_gb_total() - avail def get_hypervisor_type(self): - """ Get hypervisor type """ + """Get hypervisor type. + + :returns: hypervisor type (ex. qemu) + + """ + return self._conn.getType() def get_hypervisor_version(self): - """ Get hypervisor version """ + """Get hypervisor version. + + :returns: hypervisor version (ex. 12003) + + """ + return self._conn.getVersion() def get_cpu_info(self): - """ Get cpuinfo information """ + """Get cpuinfo information. + + Obtains cpu feature from virConnect.getCapabilities, + and returns as a json string. + + :return: see above description + + """ + xml = self._conn.getCapabilities() xml = libxml2.parseDoc(xml) nodes = xml.xpathEval('//cpu') @@ -931,17 +984,9 @@ class LibvirtConnection(object): for nodes in feature_nodes: features.append(nodes.get_properties().getContent()) - template = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ - """"topology":{"cores":"%s", "threads":"%s", """ - """"sockets":"%s"}, "features":[%s]}""") - f = ['"%s"' % x for x in features] - return template % (cpu_info['arch'], - cpu_info['model'], - cpu_info['vendor'], - topology['cores'], - topology['sockets'], - topology['threads'], - ', '.join(f)) + cpu_info['topology'] = topology + cpu_info['features'] = features + return utils.dumps(cpu_info) def block_stats(self, instance_name, disk): """ @@ -974,12 +1019,16 @@ class LibvirtConnection(object): self.firewall_driver.refresh_security_group_members(security_group_id) def update_available_resource(self, ctxt, host): - """ - Update compute manager resource info on Service table. + """Updates compute manager resource info on ComputeService table. + This method is called when nova-coompute launches, and whenever admin executes "nova-manage service update_resource". + :param ctxt: security context + :param host: hostname that compute manager is currently running + """ + try: service_ref = db.service_get_all_compute_by_host(ctxt, host)[0] except exception.NotFound: @@ -1008,44 +1057,44 @@ class LibvirtConnection(object): db.compute_service_update(ctxt, compute_service_ref[0]['id'], dic) def compare_cpu(self, cpu_info): - """ - Check the host cpu is compatible to a cpu given by xml. + """Checks the host cpu is compatible to a cpu given by xml. + "xml" must be a part of libvirt.openReadonly().getCapabilities(). return values follows by virCPUCompareResult. if 0 > return value, do live migration. - 'http://libvirt.org/html/libvirt-libvirt.html#virCPUCompareResult' + + :param cpu_info: json string that shows cpu feature(see get_cpu_info()) + :returns: + None. if given cpu info is not compatible to this server, + raise exception. + """ - msg = _('Checking cpu_info: instance was launched this cpu.\n: %s ') - LOG.info(msg % cpu_info) - dic = json.loads(cpu_info) - xml = str(Template(self.cpuinfo_xml, searchList=dic)) - msg = _('to xml...\n: %s ') - LOG.info(msg % xml) - url = 'http://libvirt.org/html/libvirt-libvirt.html' - url += '#virCPUCompareResult\n' - msg = 'CPU does not have compativility.\n' - msg += 'result:%s \n' - msg += 'Refer to %s' - msg = _(msg) + LOG.info(_('Checking cpu_info: instance was launched this cpu.\n%s') + % cpu_info) + dic = utils.loads(cpu_info) + xml = str(Template(self.cpuinfo_xml, searchList=dic)) + LOG.info(_('to xml...\n:%s ' % xml)) + u = "http://libvirt.org/html/libvirt-libvirt.html#virCPUCompareResult" + m = _("CPU doesn't have compatibility.\n\n%(ret)s\n\nRefer to %(u)s") # unknown character exists in xml, then libvirt complains try: ret = self._conn.compareCPU(xml, 0) except libvirt.libvirtError, e: - LOG.error(msg % (e.message, url)) - raise e + ret = e.message + LOG.error(m % locals()) + raise if ret <= 0: - raise exception.Invalid(msg % (ret, url)) + raise exception.Invalid(m % locals()) return def ensure_filtering_rules_for_instance(self, instance_ref): - """ - Setting up inevitable filtering rules on compute node, - and waiting for its completion. + """Setting up filtering rules and waiting for its completion. + To migrate an instance, filtering rules to hypervisors and firewalls are inevitable on destination host. ( Waiting only for filterling rules to hypervisor, @@ -1062,9 +1111,12 @@ class LibvirtConnection(object): Don't use thread for this method since migration should not be started when setting-up filtering rules operations are not completed. + + :params instance_ref: nova.db.sqlalchemy.models.Instance object + """ - # Tf any instances never launch at destination host, + # If any instances never launch at destination host, # basic-filtering must be set here. self.firewall_driver.setup_basic_filtering(instance_ref) # setting up n)ova-instance-instance-xx mainly. @@ -1088,16 +1140,42 @@ class LibvirtConnection(object): def live_migration(self, ctxt, instance_ref, dest, post_method, recover_method): + """Spawning live_migration operation for distributing high-load. + + :params ctxt: security context + :params instance_ref: + nova.db.sqlalchemy.models.Instance object + instance object that is migrated. + :params dest: destination host + :params post_method: + post operation method. + expected nova.compute.manager.post_live_migration. + :params recover_method: + recovery method when any exception occurs. + expected nova.compute.manager.recover_live_migration. + """ - Just spawning live_migration operation for - distributing high-load. - """ + greenthread.spawn(self._live_migration, ctxt, instance_ref, dest, post_method, recover_method) def _live_migration(self, ctxt, instance_ref, dest, post_method, recover_method): - """ Do live migration.""" + """Do live migration. + + :params ctxt: security context + :params instance_ref: + nova.db.sqlalchemy.models.Instance object + instance object that is migrated. + :params dest: destination host + :params post_method: + post operation method. + expected nova.compute.manager.post_live_migration. + :params recover_method: + recovery method when any exception occurs. + expected nova.compute.manager.recover_live_migration. + + """ # Do live migration. try: @@ -1122,7 +1200,7 @@ class LibvirtConnection(object): except Exception, e: recover_method(ctxt, instance_ref) - raise e + raise # Waiting for completion of live_migration. timer = utils.LoopingCall(f=None) @@ -1139,7 +1217,7 @@ class LibvirtConnection(object): timer.start(interval=0.5, now=True) def unfilter_instance(self, instance_ref): - """See comments of same method in firewall_driver""" + """See comments of same method in firewall_driver.""" self.firewall_driver.unfilter_instance(instance_ref) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 3dd7d6e94..0e12a4587 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -231,25 +231,25 @@ class XenAPIConnection(object): 'password': FLAGS.xenapi_connection_password} def update_available_resource(self, ctxt, host): - """This method is supported only libvirt. """ + """This method is supported only by libvirt.""" return def compare_cpu(self, xml): - """This method is supported only libvirt..""" - raise NotImplementedError('This method is supported only libvirt.') + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') def ensure_filtering_rules_for_instance(self, instance_ref): - """This method is supported only libvirt..""" + """This method is supported only libvirt.""" return def live_migration(self, context, instance_ref, dest, post_method, recover_method): - """This method is supported only libvirt..""" + """This method is supported only by libvirt.""" return def unfilter_instance(self, instance_ref): - """This method is supported only libvirt..""" - raise NotImplementedError('This method is supported only libvirt.') + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') class XenAPISession(object): diff --git a/nova/volume/driver.py b/nova/volume/driver.py index b74c74197..fbc52a598 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -140,7 +140,7 @@ class VolumeDriver(object): def check_for_export(self, context, volume_id): """Make sure volume is exported.""" - return True + raise NotImplementedError() class AOEDriver(VolumeDriver): @@ -229,9 +229,9 @@ class AOEDriver(VolumeDriver): break if not exported: # Instance will be terminated in this case. - desc = _("""Cannot confirm exported volume id:%(volume_id)s.""" - """vblade process for e%(shelf_id)s.%(blade_id)s """ - """isn't running.""") % locals() + desc = _("Cannot confirm exported volume id:%(volume_id)s." + "vblade process for e%(shelf_id)s.%(blade_id)s " + "isn't running.") % locals() raise exception.ProcessExecutionError(out, _err, cmd=cmd, description=desc) @@ -373,9 +373,9 @@ class ISCSIDriver(VolumeDriver): # Instances remount read-only in this case. # /etc/init.d/iscsitarget restart and rebooting nova-volume # is better since ensure_export() works at boot time. - logging.error(_("""Cannot confirm exported volume """ - """id:%(volume_id)s.""") % locals()) - raise e + logging.error(_("Cannot confirm exported volume " + "id:%(volume_id)s.") % locals()) + raise class FakeISCSIDriver(ISCSIDriver): -- cgit From c6b2d07f47004576fa386a6d270203b1d7937664 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Wed, 23 Feb 2011 00:15:39 +0900 Subject: Fix tiny mitakes! (remove unnecessary comment, etc) --- nova/tests/test_scheduler.py | 2 +- nova/tests/test_volume.py | 3 --- nova/virt/libvirt_conn.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) (limited to 'nova') diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 301106848..47a6d0e82 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -258,10 +258,10 @@ class SimpleDriverTestCase(test.TestCase): inst['project_id'] = self.project.id inst['instance_type'] = 'm1.tiny' inst['mac_address'] = utils.generate_mac() + inst['vcpus'] = kwargs.get('vcpus', 1) inst['ami_launch_index'] = 0 inst['availability_zone'] = kwargs.get('availability_zone', None) inst['host'] = kwargs.get('host', 'dummy') - inst['vcpus'] = kwargs.get('vcpus', 4) inst['memory_mb'] = kwargs.get('memory_mb', 20) inst['local_gb'] = kwargs.get('local_gb', 30) inst['launched_on'] = kwargs.get('launghed_on', 'dummy') diff --git a/nova/tests/test_volume.py b/nova/tests/test_volume.py index 6ae075caa..e8b4ceee8 100644 --- a/nova/tests/test_volume.py +++ b/nova/tests/test_volume.py @@ -318,9 +318,6 @@ class ISCSITestCase(DriverTestCase): mountpoint = "/dev/sd" + chr((ord('b') + index)) db.volume_attached(self.context, vol_ref['id'], self.instance_id, mountpoint) - #iscsi_target = db.volume_allocate_iscsi_target(self.context, - # vol_ref['id'], - # vol_ref['host']) volume_id_list.append(vol_ref['id']) return volume_id_list diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 934aed960..118ea13e5 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -888,7 +888,7 @@ class LibvirtConnection(object): :returns: The total number of vcpu that currently used. - """ + """ total = 0 for i in self._conn.listDomainsID(): -- cgit From 485a6c5a9502679bc5ecf02f8e758170ac0335dc Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Wed, 23 Feb 2011 01:20:39 +0900 Subject: Fixed some docstring --- nova/scheduler/manager.py | 2 +- nova/tests/test_scheduler.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'nova') diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 783594c6f..cd5012fd5 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -59,7 +59,7 @@ class SchedulerManager(manager.Manager): try: host = getattr(self.driver, driver_method)(elevated, *args, **kwargs) - except AttributeError, e: + except AttributeError: host = self.driver.schedule(elevated, topic, *args, **kwargs) rpc.cast(context, diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 47a6d0e82..71e524bca 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -60,6 +60,7 @@ class SchedulerTestCase(test.TestCase): self.flags(scheduler_driver='nova.tests.test_scheduler.TestDriver') def _create_compute_service(self): + """Create compute-manager(ComputeService and Service record).""" ctxt = context.get_admin_context() dic = {'host': 'dummy', 'binary': 'nova-compute', 'topic': 'compute', 'report_count': 0, 'availability_zone': 'dummyzone'} @@ -150,7 +151,7 @@ class SchedulerTestCase(test.TestCase): db.service_destroy(ctxt, s_ref['id']) def test_show_host_resources_works_correctly(self): - """show_host_resources() works correctly as expected.""" + """Show_host_resources() works correctly as expected.""" scheduler = manager.SchedulerManager() ctxt = context.get_admin_context() -- cgit From 8b0e8b155eab313e0caece48eee609d12df5e5d4 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 22 Feb 2011 20:45:36 +0100 Subject: Rename "SNATTING" chain to "snat". --- nova/network/linux_net.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 849181641..34337901a 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -208,7 +208,7 @@ class IptablesManager(object): For ipv4, the builtin PREROUTING, OUTPUT, and POSTROUTING nat chains are wrapped in the same was as the builtin filter chains. Additionally, there's - a SNATTING chain that is applied after the POSTROUTING chain. + a snat chain that is applied after the POSTROUTING chain. """ def __init__(self, execute=None): if not execute: @@ -261,16 +261,16 @@ class IptablesManager(object): self.ipv4['nat'].add_rule('POSTROUTING', '-j nova-postrouting-bottom', wrap=False) - # We add a SNATTING chain to the shared nova-postrouting-bottom chain + # We add a snat chain to the shared nova-postrouting-bottom chain # so that it's applied last. - self.ipv4['nat'].add_chain('SNATTING') - self.ipv4['nat'].add_rule('nova-postrouting-bottom', '-j $SNATTING', + self.ipv4['nat'].add_chain('snat') + self.ipv4['nat'].add_rule('nova-postrouting-bottom', '-j $snat', wrap=False) # And then we add a floating-snat chain and jump to first thing in - # the SNATTING chain. + # the snat chain. self.ipv4['nat'].add_chain('floating-snat') - self.ipv4['nat'].add_rule('SNATTING', '-j $floating-snat') + self.ipv4['nat'].add_rule('snat', '-j $floating-snat') self.semaphore = semaphore.Semaphore() @@ -376,7 +376,7 @@ def init_host(): # NOTE(devcamcar): Cloud public SNAT entries and the default # SNAT rule for outbound traffic. - iptables_manager.ipv4['nat'].add_rule("SNATTING", + iptables_manager.ipv4['nat'].add_rule("snat", "-s %s -j SNAT --to-source %s" % \ (FLAGS.fixed_range, FLAGS.routing_source_ip)) -- cgit From 2714b2df0d21ecb08966c4d145d2d75fa1bb201d Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Sun, 27 Feb 2011 00:07:03 +0100 Subject: fixed FIXME --- nova/db/sqlalchemy/api.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) (limited to 'nova') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index d07c53759..828d24c78 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -598,17 +598,11 @@ def fixed_ip_get_all(context, session=None): def fixed_ip_get_all_by_host(context, host=None): session = get_session() - # FIXME: I'm sure that SQLAlchemy can handle this in a nicer way - instances = session.query(models.Instance).\ - filter_by(state=1).\ - filter_by(host=host).\ - all() - - result = [] - for instance in instances: - result.append(session.query(models.FixedIp).\ - filter_by(instance_id=instance['id']).\ - first()) + result = session.query(models.FixedIp).\ + join(models.FixedIp.instance).\ + filter_by(state=1).\ + filter_by(host=host).\ + all() if not result: raise exception.NotFound(_('No fixed ips for this host defined')) -- cgit From 38c21546ecc079300c575e5950bcb990eecee3a3 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 27 Feb 2011 20:28:04 -0500 Subject: execute: shell=True removed. --- nova/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/utils.py b/nova/utils.py index 0cf91e0cc..40a8d8d8c 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -125,7 +125,7 @@ def fetchfile(url, target): # c.perform() # c.close() # fp.close() - execute("curl --fail %s -o %s" % (url, target)) + execute("curl","--fail",url,"-o",target) def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): @@ -133,7 +133,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + obj = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) result = None if process_input != None: @@ -254,7 +254,7 @@ def last_octet(address): def get_my_linklocal(interface): try: - if_str = execute("ip -f inet6 -o addr show %s" % interface) + if_str = execute("ip","-f","inet6","-o","addr","show", interface) condition = "\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link" links = [re.search(condition, x) for x in if_str[0].split('\n')] address = [w.group(1) for w in links if w is not None] -- cgit From 4f90783224025618661bf8814e016843ec237875 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 27 Feb 2011 20:52:32 -0500 Subject: execvp --- nova/volume/driver.py | 69 ++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 34 deletions(-) (limited to 'nova') diff --git a/nova/volume/driver.py b/nova/volume/driver.py index e3744c790..e73202b73 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -97,22 +97,21 @@ class VolumeDriver(object): sizestr = '100M' else: sizestr = '%sG' % volume['size'] - self._try_execute("sudo lvcreate -L %s -n %s %s" % - (sizestr, + self._try_execute('sudo','lvcreate','-L',sizestr,'-n', volume['name'], - FLAGS.volume_group)) + FLAGS.volume_group) def delete_volume(self, volume): """Deletes a logical volume.""" try: - self._try_execute("sudo lvdisplay %s/%s" % + self._try_execute('sudo','lvdisplay','%s/%s" % (FLAGS.volume_group, volume['name'])) except Exception as e: # If the volume isn't present, then don't attempt to delete return True - self._try_execute("sudo lvremove -f %s/%s" % + self._try_execute('sudo','lvremove','-f',"%s/%s" % (FLAGS.volume_group, volume['name'])) @@ -168,12 +167,13 @@ class AOEDriver(VolumeDriver): blade_id) = self.db.volume_allocate_shelf_and_blade(context, volume['id']) self._try_execute( - "sudo vblade-persist setup %s %s %s /dev/%s/%s" % - (shelf_id, + 'sudo','vblade-persist','setup', + shelf_id, blade_id, FLAGS.aoe_eth_dev, - FLAGS.volume_group, - volume['name'])) + "/dev/%s/%s" % + (FLAGS.volume_group, + volume['name'])) # NOTE(vish): The standard _try_execute does not work here # because these methods throw errors if other # volumes on this host are in the process of @@ -182,9 +182,9 @@ class AOEDriver(VolumeDriver): # just wait a bit for the current volume to # be ready and ignore any errors. time.sleep(2) - self._execute("sudo vblade-persist auto all", + self._execute('sudo','vblade-persist','auto','all', check_exit_code=False) - self._execute("sudo vblade-persist start all", + self._execute('sudo','vblade-persist','start','all', check_exit_code=False) def remove_export(self, context, volume): @@ -192,15 +192,15 @@ class AOEDriver(VolumeDriver): (shelf_id, blade_id) = self.db.volume_get_shelf_and_blade(context, volume['id']) - self._try_execute("sudo vblade-persist stop %s %s" % - (shelf_id, blade_id)) - self._try_execute("sudo vblade-persist destroy %s %s" % - (shelf_id, blade_id)) + self._try_execute('sudo','vblade-persist','stop', + shelf_id, blade_id) + self._try_execute('sudo','vblade-persist','destroy', + shelf_id, blade_id) def discover_volume(self, _volume): """Discover volume on a remote host.""" - self._execute("sudo aoe-discover") - self._execute("sudo aoe-stat", check_exit_code=False) + self._execute('sudo','aoe-discover') + self._execute('sudo','aoe-stat', check_exit_code=False) def undiscover_volume(self, _volume): """Undiscover volume on a remote host.""" @@ -252,13 +252,16 @@ class ISCSIDriver(VolumeDriver): iscsi_name = "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) volume_path = "/dev/%s/%s" % (FLAGS.volume_group, volume['name']) - self._sync_exec("sudo ietadm --op new " - "--tid=%s --params Name=%s" % - (iscsi_target, iscsi_name), + self._sync_exec('sudo','ietadm','--op','new', + "--tid=%s" % iscsi_target, + '--params', + "Name=%s" % iscsi-name, check_exit_code=False) - self._sync_exec("sudo ietadm --op new --tid=%s " - "--lun=0 --params Path=%s,Type=fileio" % - (iscsi_target, volume_path), + self._sync_exec('sudo','ietadm','--op','new', + "--tid=%s" % iscsi_target, + '--lun=0', + '--params', + "Path=%s,Type=fileio" % volume_path, check_exit_code=False) def _ensure_iscsi_targets(self, context, host): @@ -490,16 +493,13 @@ class RBDDriver(VolumeDriver): size = 100 else: size = int(volume['size']) * 1024 - self._try_execute("rbd --pool %s --size %d create %s" % - (FLAGS.rbd_pool, - size, - volume['name'])) + self._try_execute('rbd','--pool',FLAGS.rbd_pool, + '--size', size,'create', volume['name']) def delete_volume(self, volume): """Deletes a logical volume.""" - self._try_execute("rbd --pool %s rm %s" % - (FLAGS.rbd_pool, - volume['name'])) + self._try_execute('rbd','--pool',FLAGS.rbd_pool, + 'rm', voluname['name']) def local_path(self, volume): """Returns the path of the rbd volume.""" @@ -534,7 +534,7 @@ class SheepdogDriver(VolumeDriver): def check_for_setup_error(self): """Returns an error if prerequisites aren't met""" try: - (out, err) = self._execute("collie cluster info") + (out, err) = self._execute('collie','cluster','info') if not out.startswith('running'): raise exception.Error(_("Sheepdog is not working: %s") % out) except exception.ProcessExecutionError: @@ -546,12 +546,13 @@ class SheepdogDriver(VolumeDriver): sizestr = '100M' else: sizestr = '%sG' % volume['size'] - self._try_execute("qemu-img create sheepdog:%s %s" % - (volume['name'], sizestr)) + self._try_execute('qemu-img','create', + "sheepdog:%s" %s" % volume['name'], + sizestr) def delete_volume(self, volume): """Deletes a logical volume""" - self._try_execute("collie vdi delete %s" % volume['name']) + self._try_execute('collie','vdi','delete',volume['name']) def local_path(self, volume): return "sheepdog:%s" % volume['name'] -- cgit From 953efce36b74c18a32ef9c42e6b1a57190e3ff6e Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 27 Feb 2011 20:53:53 -0500 Subject: execvp --- nova/crypto.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) (limited to 'nova') diff --git a/nova/crypto.py b/nova/crypto.py index a34b940f5..b240a3958 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -105,8 +105,8 @@ def generate_key_pair(bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.join(tmpdir, 'temp') - utils.execute('ssh-keygen -q -b %d -N "" -f %s' % (bits, keyfile)) - (out, err) = utils.execute('ssh-keygen -q -l -f %s.pub' % (keyfile)) + utils.execute('ssh-keygen','-q','-b',"%d" % bits,'-N','""','-f',keyfile) + (out, err) = utils.execute('ssh-keygen','-q','-l','-f',"%s.pub" % (keyfile)) fingerprint = out.split(' ')[1] private_key = open(keyfile).read() public_key = open(keyfile + '.pub').read() @@ -118,7 +118,7 @@ def generate_key_pair(bits=1024): # bio = M2Crypto.BIO.MemoryBuffer() # key.save_pub_key_bio(bio) # public_key = bio.read() - # public_key, err = execute('ssh-keygen -y -f /dev/stdin', private_key) + # public_key, err = execute('ssh-keygen','-y','-f','/dev/stdin', private_key) return (private_key, public_key, fingerprint) @@ -143,8 +143,8 @@ def revoke_cert(project_id, file_name): start = os.getcwd() os.chdir(ca_folder(project_id)) # NOTE(vish): potential race condition here - utils.execute("openssl ca -config ./openssl.cnf -revoke '%s'" % file_name) - utils.execute("openssl ca -gencrl -config ./openssl.cnf -out '%s'" % + utils.execute('openssl','ca','-config','./openssl.cnf','-revoke',"'%s'" % file_name) + utils.execute('openssl','ca','-gencrl','-config','./openssl.cnf','-out',"'%s'" % FLAGS.crl_file) os.chdir(start) @@ -193,9 +193,8 @@ def generate_x509_cert(user_id, project_id, bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key')) csrfile = os.path.join(tmpdir, 'temp.csr') - utils.execute("openssl genrsa -out %s %s" % (keyfile, bits)) - utils.execute("openssl req -new -key %s -out %s -batch -subj %s" % - (keyfile, csrfile, subject)) + utils.execute('openssl','genrsa','-out',keyfile,bits) + utils.execute('openssl','req','-new','-key',keyfile,'-out',csrfile,'-batch','-subj',subject) private_key = open(keyfile).read() csr = open(csrfile).read() shutil.rmtree(tmpdir) @@ -212,8 +211,7 @@ def _ensure_project_folder(project_id): if not os.path.exists(ca_path(project_id)): start = os.getcwd() os.chdir(ca_folder()) - utils.execute("sh geninter.sh %s %s" % - (project_id, _project_cert_subject(project_id))) + utils.execute('sh','geninter.sh',project_id, _project_cert_subject(project_id)) os.chdir(start) @@ -228,8 +226,8 @@ def generate_vpn_files(project_id): start = os.getcwd() os.chdir(ca_folder()) # TODO(vish): the shell scripts could all be done in python - utils.execute("sh genvpn.sh %s %s" % - (project_id, _vpn_cert_subject(project_id))) + utils.execute('sh','genvpn.sh', + project_id, _vpn_cert_subject(project_id)) with open(csr_fn, "r") as csrfile: csr_text = csrfile.read() (serial, signed_csr) = sign_csr(csr_text, project_id) @@ -259,9 +257,9 @@ def _sign_csr(csr_text, ca_folder): start = os.getcwd() # Change working dir to CA os.chdir(ca_folder) - utils.execute("openssl ca -batch -out %s -config " - "./openssl.cnf -infiles %s" % (outbound, inbound)) - out, _err = utils.execute("openssl x509 -in %s -serial -noout" % outbound) + utils.execute('openssl','ca','-batch','-out',outbound,'-config' + './openssl.cnf','-infiles',inbound) + out, _err = utils.execute('openssl','x509','-in',outbound','-serial','-noout') serial = out.rpartition("=")[2] os.chdir(start) with open(outbound, "r") as crtfile: -- cgit From 90abcdc7ae9e3f855dadb1ccc88892a2cc7bab05 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 27 Feb 2011 20:57:13 -0500 Subject: execvp --- nova/console/xvp.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'nova') diff --git a/nova/console/xvp.py b/nova/console/xvp.py index cd257e0a6..271dffa54 100644 --- a/nova/console/xvp.py +++ b/nova/console/xvp.py @@ -133,10 +133,10 @@ class XVPConsoleProxy(object): return logging.debug(_("Starting xvp")) try: - utils.execute('xvp -p %s -c %s -l %s' % - (FLAGS.console_xvp_pid, - FLAGS.console_xvp_conf, - FLAGS.console_xvp_log)) + utils.execute('xvp', + '-p',FLAGS.console_xvp_pid, + '-c',FLAGS.console_xvp_conf, + '-l',FLAGS.console_xvp_log) except exception.ProcessExecutionError, err: logging.error(_("Error starting xvp: %s") % err) @@ -190,5 +190,5 @@ class XVPConsoleProxy(object): flag = '-x' #xvp will blow up on passwords that are too long (mdragon) password = password[:maxlen] - out, err = utils.execute('xvp %s' % flag, process_input=password) + out, err = utils.execute('xvp', flag, process_input=password) return out.strip() -- cgit From 8b3e9ad11c2f5c425701f1eb4abb7b3f577ae1cc Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 28 Feb 2011 12:37:02 +0100 Subject: Add utils.synchronized decorator to allow for synchronising method entrance across multiple workers on the same host. --- nova/tests/test_misc.py | 37 ++++++++++++++++++++++++++++++++++++- nova/utils.py | 11 +++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index e6da6112a..154b6fae6 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -14,10 +14,14 @@ # License for the specific language governing permissions and limitations # under the License. +from datetime import datetime +import errno import os +import select +import time from nova import test -from nova.utils import parse_mailmap, str_dict_replace +from nova.utils import parse_mailmap, str_dict_replace, synchronized class ProjectTestCase(test.TestCase): @@ -55,3 +59,34 @@ class ProjectTestCase(test.TestCase): '%r not listed in Authors' % missing) finally: tree.unlock() + + +class LockTestCase(test.TestCase): + def test_synchronized(self): + rpipe, wpipe = os.pipe() + pid = os.fork() + if pid > 0: + os.close(wpipe) + + @synchronized('testlock') + def f(): + rfds, _, __ = select.select([rpipe], [], [], 1) + self.assertEquals(len(rfds), 0, "The other process, which was" + " supposed to be locked, " + "wrote on its end of the " + "pipe") + os.close(rpipe) + + f() + else: + os.close(rpipe) + + @synchronized('testlock') + def g(): + try: + os.write(wpipe, "foo") + except OSError, e: + self.assertEquals(e.errno, errno.EPIPE) + return + g() + os._exit(0) diff --git a/nova/utils.py b/nova/utils.py index 0cf91e0cc..cb1ea5a7d 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -25,6 +25,7 @@ import base64 import datetime import inspect import json +import lockfile import os import random import socket @@ -491,6 +492,16 @@ def loads(s): return json.loads(s) +def synchronized(name): + def wrap(f): + def inner(*args, **kwargs): + lock = lockfile.FileLock('nova-%s.lock' % name) + with lock: + return f(*args, **kwargs) + return inner + return wrap + + def ensure_b64_encoding(val): """Safety method to ensure that values expected to be base64-encoded actually are. If they are, the value is returned unchanged. Otherwise, -- cgit From 0d3a1f9c7478ae3c4f042e1876d47359804e1973 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 28 Feb 2011 15:29:42 +0100 Subject: Wrap IptablesManager.apply() calls in utils.synchronized to avoid having different workers step on each other's toes. --- nova/network/linux_net.py | 1 + 1 file changed, 1 insertion(+) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 34337901a..8832f7c68 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -274,6 +274,7 @@ class IptablesManager(object): self.semaphore = semaphore.Semaphore() + @utils.synchronized('iptables') def apply(self): """Apply the current in-memory set of iptables rules -- cgit From d5736e925f288462f6325130be0af49f0ace5884 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 28 Feb 2011 23:31:09 +0100 Subject: Add a lock_path flag for lock files. --- nova/flags.py | 2 ++ nova/utils.py | 5 ++++- 2 files changed, 6 insertions(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/flags.py b/nova/flags.py index 8cf199b2f..213d4d4e1 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -321,6 +321,8 @@ DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger') DEFINE_string('state_path', os.path.join(os.path.dirname(__file__), '../'), "Top-level directory for maintaining nova's state") +DEFINE_string('lock_path', os.path.join(os.path.dirname(__file__), '../'), + "Directory for lock files") DEFINE_string('logdir', None, 'output to a per-service log file in named ' 'directory') diff --git a/nova/utils.py b/nova/utils.py index cb1ea5a7d..48f12350f 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -44,11 +44,13 @@ from eventlet.green import subprocess from nova import exception from nova.exception import ProcessExecutionError +from nova import flags from nova import log as logging LOG = logging.getLogger("nova.utils") TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +FLAGS = flags.FLAGS def import_class(import_str): @@ -495,7 +497,8 @@ def loads(s): def synchronized(name): def wrap(f): def inner(*args, **kwargs): - lock = lockfile.FileLock('nova-%s.lock' % name) + lock = lockfile.FileLock(os.path.join(FLAGS.lock_path, + 'nova-%s.lock' % name)) with lock: return f(*args, **kwargs) return inner -- cgit From 0550124fcd863be60dd0e6fefb5f30641331b198 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Mon, 28 Feb 2011 18:06:11 -0500 Subject: add test for instance creation without personalities --- nova/tests/api/openstack/test_servers.py | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 7a25abe9d..7e5bc0080 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -232,6 +232,46 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) + def _create_instance_with_personality(self, personality): + + class FakeComputeAPI(object): + + def __init__(self): + self.onset_files = None + + def create(*args, **kwargs): + if 'onset_files' in kwargs: + self.onset_files = kwargs['onset_files'] + else: + self.onset_files = None + return [{'id': '1234', 'display_name': 'fakeinstance'}] + + def make_stub_method(canned_return): + def stub_method(*args, **kwargs): + return canned_return + return stub_method + + compute_api = FakeComputeAPI() + self.stubs.Set(nova.compute, 'API', make_stub_method(compute_api)) + self.stubs.Set(nova.api.openstack.servers.Controller, + '_get_kernel_ramdisk_from_image', make_stub_method((1, 1))) + self.stubs.Set(nova.api.openstack.common, + 'get_image_id_from_image_hash', make_stub_method(2)) + body = dict(server=dict( + name='server_test', imageId=2, flavorId=2, + metadata={}, + personality=personality)) + + req = webob.Request.blank('/v1.0/servers') + req.method = 'POST' + req.body = json.dumps(body) + return req.get_response(fakes.wsgi_app()), compute_api.onset_files + + def test_create_instance_with_no_personality(self): + res, onset_files = self._create_instance_with_personality(personality={}) + self.assertEquals(res.status_int, 200) + self.assertEquals(onset_files, None) + def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' -- cgit From 1caa7f189827b4721c2e9d3ddf753acd749d7916 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Tue, 1 Mar 2011 17:52:46 +0900 Subject: rename db migration script --- nova/compute/manager.py | 2 +- .../versions/007_add_live_migration.py | 83 ++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/007_add_live_migration.py (limited to 'nova') diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 8b90ffbca..d085a0b6a 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -818,7 +818,7 @@ class ComputeManager(manager.Manager): LOG.info(_('Migrating %(i_name)s to %(dest)s finishes successfully.') % locals()) - LOG.info(_("The below error is normally occurs." + LOG.info(_("The below error is normally occurs. " "Just check if instance is successfully migrated.\n" "libvir: QEMU error : Domain not found: no domain " "with matching name..")) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/007_add_live_migration.py b/nova/db/sqlalchemy/migrate_repo/versions/007_add_live_migration.py new file mode 100644 index 000000000..2689b5b74 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/007_add_live_migration.py @@ -0,0 +1,83 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from migrate import * +from nova import log as logging +from sqlalchemy import * + + +meta = MetaData() + +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +# +# New Tables +# + +compute_services = Table('compute_services', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('service_id', Integer(), nullable=False), + + Column('vcpus', Integer(), nullable=False), + Column('memory_mb', Integer(), nullable=False), + Column('local_gb', Integer(), nullable=False), + Column('vcpus_used', Integer(), nullable=False), + Column('memory_mb_used', Integer(), nullable=False), + Column('local_gb_used', Integer(), nullable=False), + Column('hypervisor_type', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=False), + Column('hypervisor_version', Integer(), nullable=False), + Column('cpu_info', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=False), + ) + + +# +# Tables to alter +# +instances_launched_on = Column( + 'launched_on', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=True) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + try: + compute_services.create() + except Exception: + logging.info(repr(compute_services)) + logging.exception('Exception while creating table') + meta.drop_all(tables=[compute_services]) + raise + + instances.create_column(instances_launched_on) -- cgit From d13a623625a56a029f9dd5ccba3e70f492efdb2c Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Tue, 1 Mar 2011 18:32:57 +0900 Subject: test_compute is changed b/c lack of import instance_types --- nova/tests/test_compute.py | 1 + 1 file changed, 1 insertion(+) (limited to 'nova') diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 3c88d186d..2a18dd47b 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -32,6 +32,7 @@ from nova import rpc from nova import test from nova import utils from nova.auth import manager +from nova.compute import instance_types from nova.compute import manager as compute_manager from nova.compute import power_state from nova.db.sqlalchemy import models -- cgit From 688acacd85e07fc578c8731df6a4421e64499c8b Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Tue, 1 Mar 2011 18:53:02 +0900 Subject: At previous commit, I forget to erase conflict - fixed it. --- nova/tests/test_compute.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) (limited to 'nova') diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 75fbc9324..3438719f4 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -309,7 +309,13 @@ class ComputeTestCase(test.TestCase): self.compute.terminate_instance(self.context, instance_id) -<<<<<<< TREE + def test_get_by_flavor_id(self): + type = instance_types.get_by_flavor_id(1) + self.assertEqual(type, 'm1.tiny') + + type = instance_types.get_by_flavor_id("1") + self.assertEqual(type, 'm1.tiny') + def _setup_other_managers(self): self.volume_manager = utils.import_object(FLAGS.volume_manager) self.network_manager = utils.import_object(FLAGS.network_manager) @@ -562,11 +568,3 @@ class ComputeTestCase(test.TestCase): db.instance_destroy(c, instance_id) db.volume_destroy(c, v_ref['id']) db.floating_ip_destroy(c, flo_addr) -======= - def test_get_by_flavor_id(self): - type = instance_types.get_by_flavor_id(1) - self.assertEqual(type, 'm1.tiny') - - type = instance_types.get_by_flavor_id("1") - self.assertEqual(type, 'm1.tiny') ->>>>>>> MERGE-SOURCE -- cgit From f9d08c16d5c620c711d962a78be3a94b99364f14 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 1 Mar 2011 13:56:33 -0500 Subject: support adding a single personality in the osapi --- nova/api/openstack/servers.py | 22 ++++++++++++++++++++-- nova/tests/api/openstack/test_servers.py | 21 ++++++++++++++++----- 2 files changed, 36 insertions(+), 7 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 73c7bfe17..92e5c9024 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -80,7 +80,6 @@ def _translate_detail_keys(inst): return dict(server=inst_dict) - def _translate_keys(inst): """ Coerces into dictionary format, excluding all model attributes save for id and name """ @@ -154,6 +153,22 @@ class Controller(wsgi.Controller): image = self._image_service.show(req.environ['nova.context'], image_id) return lookup('kernel_id'), lookup('ramdisk_id') + + def _get_onset_files_from_personality_attr(self, personality_attr): + """ + Create a list of onset files from the personality request attribute + + At this time, onset_files must be formatted as a list of + (file_path, file_content) pairs for compatibility with the + underlying compute service. + """ + onset_files = [] + for personality in personality_attr: + path = personality['path'] + contents = personality['contents'] + onset_files.append((path, contents)) + return onset_files + def create(self, req): """ Creates a new server for a given user """ env = self._deserialize(req.body, req) @@ -181,6 +196,9 @@ class Controller(wsgi.Controller): for k, v in env['server']['metadata'].items(): metadata.append({'key': k, 'value': v}) + personality = env['server'].get('personality', []) + onset_files = self._get_onset_files_from_personality_attr(personality) + instances = self.compute_api.create( context, instance_types.get_by_flavor_id(env['server']['flavorId']), @@ -192,7 +210,7 @@ class Controller(wsgi.Controller): key_name=key_pair['name'], key_data=key_pair['public_key'], metadata=metadata, - onset_files=env.get('onset_files', [])) + onset_files=onset_files) return _translate_keys(instances[0]) def update(self, req, id): diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 7e5bc0080..42665ba6d 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -232,6 +232,9 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) + def _personality_dict(self, path, contents): + return {'path': path, 'contents': contents} + def _create_instance_with_personality(self, personality): class FakeComputeAPI(object): @@ -239,7 +242,7 @@ class ServersTest(test.TestCase): def __init__(self): self.onset_files = None - def create(*args, **kwargs): + def create(self, *args, **kwargs): if 'onset_files' in kwargs: self.onset_files = kwargs['onset_files'] else: @@ -265,12 +268,20 @@ class ServersTest(test.TestCase): req = webob.Request.blank('/v1.0/servers') req.method = 'POST' req.body = json.dumps(body) - return req.get_response(fakes.wsgi_app()), compute_api.onset_files + return req, req.get_response(fakes.wsgi_app()), compute_api.onset_files def test_create_instance_with_no_personality(self): - res, onset_files = self._create_instance_with_personality(personality={}) - self.assertEquals(res.status_int, 200) - self.assertEquals(onset_files, None) + request, response, onset_files = \ + self._create_instance_with_personality(personality=[]) + self.assertEquals(response.status_int, 200) + self.assertEquals(onset_files, []) + + def test_create_instance_with_one_personality(self): + personality = [self._personality_dict('/my/path', 'myfilecontents')] + request, response, onset_files = \ + self._create_instance_with_personality(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(onset_files, [('/my/path', 'myfilecontents')]) def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') -- cgit From be9004ffa4c70358c8edda1f33ffe7ba7e1ae1ee Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 1 Mar 2011 20:49:46 +0100 Subject: Use functools.wraps to make sure wrapped method's metadata (docstring and name) doesn't get mangled. --- nova/tests/test_misc.py | 12 ++++++++++-- nova/utils.py | 6 ++++-- 2 files changed, 14 insertions(+), 4 deletions(-) (limited to 'nova') diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index 154b6fae6..9f572b58e 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -14,11 +14,9 @@ # License for the specific language governing permissions and limitations # under the License. -from datetime import datetime import errno import os import select -import time from nova import test from nova.utils import parse_mailmap, str_dict_replace, synchronized @@ -62,6 +60,16 @@ class ProjectTestCase(test.TestCase): class LockTestCase(test.TestCase): + def test_synchronized_wrapped_function_metadata(self): + @synchronized('whatever') + def foo(): + """Bar""" + pass + self.assertEquals(foo.__doc__, 'Bar', "Wrapped function's docstring " + "got lost") + self.assertEquals(foo.__name__, 'foo', "Wrapped function's name " + "got mangled") + def test_synchronized(self): rpipe, wpipe = os.pipe() pid = os.fork() diff --git a/nova/utils.py b/nova/utils.py index 48f12350f..9929e6fef 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -23,11 +23,14 @@ System-level utilities and helper functions. import base64 import datetime +import functools import inspect import json import lockfile +import netaddr import os import random +import re import socket import string import struct @@ -35,8 +38,6 @@ import sys import time import types from xml.sax import saxutils -import re -import netaddr from eventlet import event from eventlet import greenthread @@ -496,6 +497,7 @@ def loads(s): def synchronized(name): def wrap(f): + @functools.wraps(f) def inner(*args, **kwargs): lock = lockfile.FileLock(os.path.join(FLAGS.lock_path, 'nova-%s.lock' % name)) -- cgit From 94e42c3002f9043fc3c5b90a1cb5ad0c50ba261b Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 1 Mar 2011 16:21:37 -0500 Subject: ensure personality contents are b64 encoded --- nova/api/openstack/servers.py | 15 ++++++++++----- nova/tests/api/openstack/test_servers.py | 21 +++++++++++++++++---- 2 files changed, 27 insertions(+), 9 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 8491fe697..8908bbdca 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import base64 import hashlib import json import traceback @@ -138,7 +139,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def _get_onset_files_from_personality_attr(self, personality_attr): + def _get_onset_files_from_personality(self, personality): """ Create a list of onset files from the personality request attribute @@ -147,9 +148,13 @@ class Controller(wsgi.Controller): underlying compute service. """ onset_files = [] - for personality in personality_attr: - path = personality['path'] - contents = personality['contents'] + for item in personality: + path = item['path'] + try: + contents = base64.b64decode(item['contents']) + except TypeError: + raise exc.HTTPBadRequest(explanation= + 'Personality content for %s cannot be decoded' % path) onset_files.append((path, contents)) return onset_files @@ -181,7 +186,7 @@ class Controller(wsgi.Controller): metadata.append({'key': k, 'value': v}) personality = env['server'].get('personality', []) - onset_files = self._get_onset_files_from_personality_attr(personality) + onset_files = self._get_onset_files_from_personality(personality) instances = self.compute_api.create( context, diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 144dbd4af..4b40793a7 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -15,6 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. +import base64 import datetime import json @@ -272,16 +273,28 @@ class ServersTest(test.TestCase): def test_create_instance_with_no_personality(self): request, response, onset_files = \ - self._create_instance_with_personality(personality=[]) + self._create_instance_with_personality(personality=[]) self.assertEquals(response.status_int, 200) self.assertEquals(onset_files, []) - def test_create_instance_with_one_personality(self): - personality = [self._personality_dict('/my/path', 'myfilecontents')] + def test_create_instance_with_personality(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Hello, World!"\n' + b64contents = base64.b64encode(contents) + personality = [self._personality_dict(path, b64contents)] request, response, onset_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [('/my/path', 'myfilecontents')]) + self.assertEquals(onset_files, [(path, contents)]) + + def test_create_instance_with_personality_with_non_b64_content(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Oh no!"\n' + personality = [self._personality_dict(path, contents)] + request, response, onset_files = \ + self._create_instance_with_personality(personality) + self.assertEquals(response.status_int, 400) + self.assertEquals(onset_files, None) def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') -- cgit From 7b3ccd5fd1636ebc437a89a3667e6e712004e87f Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 1 Mar 2011 16:48:01 -0500 Subject: test osapi server create with multiple personalities --- nova/tests/api/openstack/test_servers.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 4b40793a7..dd951e90c 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -296,6 +296,20 @@ class ServersTest(test.TestCase): self.assertEquals(response.status_int, 400) self.assertEquals(onset_files, None) + def test_create_instance_with_two_personalities(self): + files = [ + ('/etc/sudoers', 'ALL ALL=NOPASSWD: ALL\n'), + ('/etc/motd', 'Enjoy your root access!\n'), + ] + personality = [] + for path, content in files: + personality.append(self._personality_dict( + path, base64.b64encode(content))) + request, response, onset_files = \ + self._create_instance_with_personality(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(onset_files, files) + def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' -- cgit From cdb1b16a6019fd68a7969666d754c4007607ae53 Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Tue, 1 Mar 2011 23:18:37 +0000 Subject: * Added ability to launch XenServer instances with per-os vm-params. --- nova/compute/api.py | 5 +- .../versions/007_add_os_type_to_instances.py | 45 ++++++ nova/db/sqlalchemy/models.py | 2 + nova/virt/xenapi/vm_utils.py | 152 +++++++++++++++------ nova/virt/xenapi/vmops.py | 17 +-- 5 files changed, 164 insertions(+), 57 deletions(-) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py (limited to 'nova') diff --git a/nova/compute/api.py b/nova/compute/api.py index 625778b66..8bdf712a0 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -125,6 +125,8 @@ class API(base.Base): raise quota.QuotaError(msg, "MetadataLimitExceeded") image = self.image_service.show(context, image_id) + os_type = image['properties'].get('os_type', 'linux') + if kernel_id is None: kernel_id = image.get('kernel_id', None) if ramdisk_id is None: @@ -180,7 +182,8 @@ class API(base.Base): 'key_data': key_data, 'locked': False, 'metadata': metadata, - 'availability_zone': availability_zone} + 'availability_zone': availability_zone, + 'os_type': os_type} elevated = context.elevated() instances = [] LOG.debug(_("Going to run %s instances..."), num_instances) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py new file mode 100644 index 000000000..d6d964b95 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py @@ -0,0 +1,45 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +# FIXME(dubs) should this be not null? Maybe create as nullable, then +# populate all existing rows with 'linux', then adding not null constraint. +instances_os_type = Column('os_type', + String(length=255, convert_unicode=False, + assert_unicode=None, unicode_error=None, + _warn_on_bytestring=False), + nullable=True) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + instances.create_column(instances_os_type) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 1882efeba..b78c95e40 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -188,6 +188,8 @@ class Instance(BASE, NovaBase): locked = Column(Boolean) + os_type = Column(String(255)) + # TODO(vish): see Ewan's email about state improvements, probably # should be in a driver base class or some such # vmstate_state = running, halted, suspended, paused diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 6d9aeb060..11f1fabe9 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -80,62 +80,80 @@ class VMHelper(HelperBase): """ @classmethod - def create_vm(cls, session, instance, kernel, ramdisk, pv_kernel=False): + def create_vm(cls, session, instance, kernel, ramdisk, use_pv_kernel=False): """Create a VM record. Returns a Deferred that gives the new VM reference. - the pv_kernel flag indicates whether the guest is HVM or PV + the use_pv_kernel flag indicates whether the guest is HVM or PV + + There are 3 scenarios: + + 1. Using paravirtualization, kernel passed in + + 2. Using paravirtualization, kernel within the image + + 3. Using hardware virtualization """ instance_type = instance_types.INSTANCE_TYPES[instance.instance_type] mem = str(long(instance_type['memory_mb']) * 1024 * 1024) vcpus = str(instance_type['vcpus']) rec = { - 'name_label': instance.name, - 'name_description': '', + 'actions_after_crash': 'destroy', + 'actions_after_reboot': 'restart', + 'actions_after_shutdown': 'destroy', + 'affinity': '', + 'blocked_operations': {}, + 'ha_always_run': False, + 'ha_restart_priority': '', + 'HVM_boot_params': {}, + 'HVM_boot_policy': '', 'is_a_template': False, - 'memory_static_min': '0', - 'memory_static_max': mem, 'memory_dynamic_min': mem, 'memory_dynamic_max': mem, - 'VCPUs_at_startup': vcpus, - 'VCPUs_max': vcpus, - 'VCPUs_params': {}, - 'actions_after_shutdown': 'destroy', - 'actions_after_reboot': 'restart', - 'actions_after_crash': 'destroy', - 'PV_bootloader': '', - 'PV_kernel': '', - 'PV_ramdisk': '', + 'memory_static_min': '0', + 'memory_static_max': mem, + 'memory_target': mem, + 'name_description': '', + 'name_label': instance.name, +# 'other_config': {'allowvssprovider': False}, + 'other_config': {}, + 'PCI_bus': '', + 'platform': {'acpi': 'true', 'apic': 'true', 'pae': 'true', + 'viridian': 'true', 'timeoffset': '0'}, 'PV_args': '', + 'PV_bootloader': '', 'PV_bootloader_args': '', + 'PV_kernel': '', 'PV_legacy_args': '', - 'HVM_boot_policy': '', - 'HVM_boot_params': {}, - 'platform': {}, - 'PCI_bus': '', + 'PV_ramdisk': '', 'recommendations': '', - 'affinity': '', + 'tags': [], 'user_version': '0', - 'other_config': {}, + 'VCPUs_at_startup': vcpus, + 'VCPUs_max': vcpus, + 'VCPUs_params': {}, + 'xenstore_data': {} } - #Complete VM configuration record according to the image type - #non-raw/raw with PV kernel/raw in HVM mode - if instance.kernel_id: - rec['PV_bootloader'] = '' - rec['PV_kernel'] = kernel - rec['PV_ramdisk'] = ramdisk - rec['PV_args'] = 'root=/dev/xvda1' - rec['PV_bootloader_args'] = '' - rec['PV_legacy_args'] = '' - else: - if pv_kernel: - rec['PV_args'] = 'noninteractive' - rec['PV_bootloader'] = 'pygrub' + + # Complete VM configuration record according to the image type + # non-raw/raw with PV kernel/raw in HVM mode + if use_pv_kernel: + rec['platform']['nx'] = 'false' + if instance.kernel_id: + # 1. Kernel explicitly passed in, use that + rec['PV_args'] = 'root=/dev/xvda1' + rec['PV_kernel'] = kernel + rec['PV_ramdisk'] = ramdisk else: - rec['HVM_boot_policy'] = 'BIOS order' - rec['HVM_boot_params'] = {'order': 'dc'} - rec['platform'] = {'acpi': 'true', 'apic': 'true', - 'pae': 'true', 'viridian': 'true'} + # 2. Use kernel within the image + rec['PV_args'] = 'clocksource=jiffies' + rec['PV_bootloader'] = 'pygrub' + else: + # 3. Using hardware virtualization + rec['platform']['nx'] = 'true' + rec['HVM_boot_params'] = {'order': 'dc'} + rec['HVM_boot_policy'] = 'BIOS order' + LOG.debug(_('Created VM %s...'), instance.name) vm_ref = session.call_xenapi('VM.create', rec) instance_name = instance.name @@ -497,17 +515,32 @@ class VMHelper(HelperBase): return uuid @classmethod - def lookup_image(cls, session, instance_id, vdi_ref): + def determine_is_pv(cls, session, instance_id, vdi_ref, disk_image_type, + os_type): """ - Determine if VDI is using a PV kernel + Determine whether the VM will use a paravirtualized kernel or if it + will use hardware virtualization. + + 1. Objectstore (any image type): + We use plugin to figure out whether the VDI uses PV + + 2. Glance (VHD): then we use `os_type`, raise if not set + + 3. Glance (DISK_RAW): use Pygrub to figure out if pv kernel is + available + + 4. Glance (DISK): pv is assumed """ if FLAGS.xenapi_image_service == 'glance': - return cls._lookup_image_glance(session, vdi_ref) + # 2, 3, 4: Glance + return cls._determine_is_pv_glance( + session, vdi_ref, disk_image_type, os_type) else: - return cls._lookup_image_objectstore(session, instance_id, vdi_ref) + # 1. Objecstore + return cls._determine_is_pv_objectstore(session, instance_id, vdi_ref) @classmethod - def _lookup_image_objectstore(cls, session, instance_id, vdi_ref): + def _determine_is_pv_objectstore(cls, session, instance_id, vdi_ref): LOG.debug(_("Looking up vdi %s for PV kernel"), vdi_ref) fn = "is_vdi_pv" args = {} @@ -523,9 +556,38 @@ class VMHelper(HelperBase): return pv @classmethod - def _lookup_image_glance(cls, session, vdi_ref): + def _determine_is_pv_glance(cls, session, vdi_ref, disk_image_type, + os_type): + """ + For a Glance image, determine if we need paravirtualization. + + The relevant scenarios are: + 2. Glance (VHD): then we use `os_type`, raise if not set + + 3. Glance (DISK_RAW): use Pygrub to figure out if pv kernel is + available + + 4. Glance (DISK): pv is assumed + """ + LOG.debug(_("Looking up vdi %s for PV kernel"), vdi_ref) - return with_vdi_attached_here(session, vdi_ref, True, _is_vdi_pv) + if disk_image_type == ImageType.DISK_VHD: + # 2. VHD + if os_type == 'windows': + is_pv = False + else: + is_pv = True + elif disk_image_type == ImageType.DISK_RAW: + # 3. RAW + is_pv = with_vdi_attached_here(session, vdi_ref, True, _is_vdi_pv) + elif disk_image_type == ImageType.DISK: + # 4. Disk + is_pv = True + else: + raise exception.Error(_("Unknown image format %(disk_image_type)s") + % locals()) + + return is_pv @classmethod def lookup(cls, session, i): diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index bc39aa140..abc1fb699 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -87,15 +87,7 @@ class VMOps(object): vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) - pv_kernel = False - if disk_image_type == ImageType.DISK_RAW: - #Have a look at the VDI and see if it has a PV kernel - pv_kernel = VMHelper.lookup_image(self._session, instance.id, - vdi_ref) - elif disk_image_type == ImageType.DISK_VHD: - # TODO(sirp): Assuming PV for now; this will need to be - # configurable as Windows will use HVM. - pv_kernel = True + os_type = instance.get('os_type', 'linux') kernel = None if instance.kernel_id: @@ -107,8 +99,11 @@ class VMOps(object): ramdisk = VMHelper.fetch_image(self._session, instance.id, instance.ramdisk_id, user, project, ImageType.KERNEL_RAMDISK) - vm_ref = VMHelper.create_vm(self._session, - instance, kernel, ramdisk, pv_kernel) + use_pv_kernel = VMHelper.determine_is_pv( + self._session, instance.id, vdi_ref, disk_image_type, os_type) + vm_ref = VMHelper.create_vm(self._session, instance, kernel, ramdisk, + use_pv_kernel) + VMHelper.create_vbd(self._session, vm_ref, vdi_ref, 0, True) # inject_network_info and create vifs -- cgit From 6321c5047c082bba8edf10a660fdb6a56430cc44 Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Wed, 2 Mar 2011 00:19:02 +0000 Subject: * Added first cut of migration for os_type on instances table * Track os_type when taking snapshots --- .../migrate_repo/versions/007_add_os_type_to_instances.py | 4 +++- nova/virt/xenapi/vm_utils.py | 9 ++++++--- nova/virt/xenapi/vmops.py | 4 ++-- 3 files changed, 11 insertions(+), 6 deletions(-) (limited to 'nova') diff --git a/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py index d6d964b95..21f21b040 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py @@ -34,7 +34,7 @@ instances_os_type = Column('os_type', String(length=255, convert_unicode=False, assert_unicode=None, unicode_error=None, _warn_on_bytestring=False), - nullable=True) + nullable=False) def upgrade(migrate_engine): @@ -43,3 +43,5 @@ def upgrade(migrate_engine): meta.bind = migrate_engine instances.create_column(instances_os_type) + + diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 11f1fabe9..9c0bb5579 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -303,7 +303,7 @@ class VMHelper(HelperBase): return template_vm_ref, template_vdi_uuids @classmethod - def upload_image(cls, session, instance_id, vdi_uuids, image_id): + def upload_image(cls, session, instance, vdi_uuids, image_id): """ Requests that the Glance plugin bundle the specified VDIs and push them into Glance using the specified human-friendly name. """ @@ -312,15 +312,18 @@ class VMHelper(HelperBase): logging.debug(_("Asking xapi to upload %(vdi_uuids)s as" " ID %(image_id)s") % locals()) + # TODO(dubs): os_type is currently defaulting to linux, we actually + # want to make this a NOT NULL column and require it to be specified. params = {'vdi_uuids': vdi_uuids, 'image_id': image_id, 'glance_host': FLAGS.glance_host, 'glance_port': FLAGS.glance_port, - 'sr_path': get_sr_path(session)} + 'sr_path': get_sr_path(session), + 'os_type': instance.get('os_type', 'linux')} kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'upload_vhd', kwargs) - session.wait_for_task(instance_id, task) + session.wait_for_task(instance.id, task) @classmethod def fetch_image(cls, session, instance_id, image, user, project, diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index abc1fb699..1edf39c5b 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -238,11 +238,11 @@ class VMOps(object): try: # call plugin to ship snapshot off to glance VMHelper.upload_image( - self._session, instance.id, template_vdi_uuids, image_id) + self._session, instance, template_vdi_uuids, image_id) finally: self._destroy(instance, template_vm_ref, shutdown=False, destroy_kernel_ramdisk=False) - + logging.debug(_("Finished snapshot and upload for VM %s"), instance) def reboot(self, instance): -- cgit From d33866923958b3529a812f4eef7dea4a6591a423 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 2 Mar 2011 17:36:21 -0500 Subject: add support for quotas on file injection --- nova/quota.py | 29 +++++++++++++++++++++++------ nova/tests/test_quota.py | 10 ++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) (limited to 'nova') diff --git a/nova/quota.py b/nova/quota.py index 6b52a97fa..14b388794 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -37,15 +37,22 @@ flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') flags.DEFINE_integer('quota_metadata_items', 128, 'number of metadata items allowed per instance') +flags.DEFINE_integer('quota_file_injection_max_files', 5, + 'number of files allowed during file injection') +flags.DEFINE_integer('quota_file_injection_max_file_bytes', 10 * 1024, + 'number of bytes allowed per file during file injection') def get_quota(context, project_id): - rval = {'instances': FLAGS.quota_instances, - 'cores': FLAGS.quota_cores, - 'volumes': FLAGS.quota_volumes, - 'gigabytes': FLAGS.quota_gigabytes, - 'floating_ips': FLAGS.quota_floating_ips, - 'metadata_items': FLAGS.quota_metadata_items} + rval = { + 'instances': FLAGS.quota_instances, + 'cores': FLAGS.quota_cores, + 'volumes': FLAGS.quota_volumes, + 'gigabytes': FLAGS.quota_gigabytes, + 'floating_ips': FLAGS.quota_floating_ips, + 'metadata_items': FLAGS.quota_metadata_items, + } + try: quota = db.quota_get(context, project_id) for key in rval.keys(): @@ -106,6 +113,16 @@ def allowed_metadata_items(context, num_metadata_items): return min(num_metadata_items, num_allowed_metadata_items) +def allowed_file_injection_files(context): + """Return the number of files allowed per file injection""" + return FLAGS.quota_file_injection_max_files + + +def allowed_file_injection_file_bytes(context): + """Return the number of bytes allowed per file during injection""" + return FLAGS.quota_file_injection_max_file_bytes + + class QuotaError(exception.ApiError): """Quota Exceeeded""" pass diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 1e42fddf3..48e5a5538 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -176,3 +176,13 @@ class QuotaTestCase(test.TestCase): instance_type='m1.small', image_id='fake', metadata=metadata) + + def test_allowed_file_injection_files(self): + self.assertEqual( + quota.allowed_file_injection_files(self.context), + FLAGS.quota_file_injection_max_files) + + def test_allowed_file_injection_file_bytes(self): + self.assertEqual( + quota.allowed_file_injection_file_bytes(self.context), + FLAGS.quota_file_injection_max_file_bytes) -- cgit From 668cdc96b3f6fb412b9d1d4a3780744d6b2340b1 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 3 Mar 2011 00:59:09 -0500 Subject: more rigorous testing and error handling for os api personality --- nova/api/openstack/servers.py | 8 ++++++-- nova/tests/api/openstack/test_servers.py | 25 +++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 8908bbdca..73c787828 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -149,9 +149,13 @@ class Controller(wsgi.Controller): """ onset_files = [] for item in personality: - path = item['path'] try: - contents = base64.b64decode(item['contents']) + path = item['path'] + contents = item['contents'] + except TypeError: + raise exc.HTTPBadRequest(explanation='Bad personality format') + try: + contents = base64.b64decode(contents) except TypeError: raise exc.HTTPBadRequest(explanation= 'Personality content for %s cannot be decoded' % path) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index dd951e90c..272d34e3a 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -265,6 +265,8 @@ class ServersTest(test.TestCase): name='server_test', imageId=2, flavorId=2, metadata={}, personality=personality)) + if personality is None: + del body['server']['personality'] req = webob.Request.blank('/v1.0/servers') req.method = 'POST' @@ -273,7 +275,7 @@ class ServersTest(test.TestCase): def test_create_instance_with_no_personality(self): request, response, onset_files = \ - self._create_instance_with_personality(personality=[]) + self._create_instance_with_personality(personality=None) self.assertEquals(response.status_int, 200) self.assertEquals(onset_files, []) @@ -296,10 +298,11 @@ class ServersTest(test.TestCase): self.assertEquals(response.status_int, 400) self.assertEquals(onset_files, None) - def test_create_instance_with_two_personalities(self): + def test_create_instance_with_three_personalities(self): files = [ ('/etc/sudoers', 'ALL ALL=NOPASSWD: ALL\n'), ('/etc/motd', 'Enjoy your root access!\n'), + ('/etc/dovecot.conf', 'dovecot\nconfig\nstuff\n'), ] personality = [] for path, content in files: @@ -310,6 +313,24 @@ class ServersTest(test.TestCase): self.assertEquals(response.status_int, 200) self.assertEquals(onset_files, files) + def test_create_instance_personality_empty_content(self): + path = '/my/file/path' + contents = '' + personality = [self._personality_dict(path, contents)] + request, response, onset_files = \ + self._create_instance_with_personality(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(onset_files, [(path, contents)]) + + def test_create_instance_personality_not_a_list(self): + path = '/my/file/path' + contents = 'myfilecontents' + personality = self._personality_dict(path, contents) + request, response, onset_files = \ + self._create_instance_with_personality(personality) + self.assertEquals(response.status_int, 400) + self.assertEquals(onset_files, None) + def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' -- cgit From 693e4335dbef72317147abd70bdaa10e0d174020 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Thu, 3 Mar 2011 22:54:11 +0900 Subject: Fixed based on reviewer's comments. Main changes are below. 1. Rename nova.compute.manager.ComputeManager.mktmpfile for better naming. 2. Several tests code in tests/test_virt.py are removed. Because it only works in libvirt environment. Only db-related testcode remains. --- nova/compute/manager.py | 53 ++++--- nova/scheduler/driver.py | 74 +++++---- nova/scheduler/manager.py | 18 +-- nova/tests/test_scheduler.py | 14 +- nova/tests/test_virt.py | 363 +++++++++---------------------------------- nova/virt/libvirt_conn.py | 10 +- nova/volume/manager.py | 4 +- 7 files changed, 166 insertions(+), 370 deletions(-) (limited to 'nova') diff --git a/nova/compute/manager.py b/nova/compute/manager.py index d085a0b6a..7104daa1e 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -624,11 +624,12 @@ class ComputeManager(manager.Manager): return self.driver.compare_cpu(cpu_info) @exception.wrap_exception - def mktmpfile(self, context): + def create_shared_storage_test_file(self, context): """Makes tmpfile under FLAGS.instance_path. This method enables compute nodes to recognize that they mounts - same shared storage. mktmpfile()/confirm_tmpfile is a pair. + same shared storage. (create|check|creanup)_shared_storage_test_file() + is a pair. :param context: security context :returns: tmpfile name(basename) @@ -636,26 +637,36 @@ class ComputeManager(manager.Manager): """ dirpath = FLAGS.instances_path - fd, name = tempfile.mkstemp(dir=dirpath) + fd, tmp_file = tempfile.mkstemp(dir=dirpath) LOG.debug(_("Creating tmpfile %s to notify to other " - "compute node that they mounts same storage.") % name) + "compute node that they mounts same storage.") % tmp_file) os.fdopen(fd, 'w+').close() - return os.path.basename(name) + return os.path.basename(tmp_file) @exception.wrap_exception - def confirm_tmpfile(self, context, filename): - """Confirms existence of the tmpfile given by path. + def check_shared_storage_test_file(self, context, filename): + """Confirms existence of the tmpfile under FLAGS.instances_path. :param context: security context :param filename: confirm existence of FLAGS.instances_path/thisfile - :returns: depends on os.remove() """ - p = os.path.join(FLAGS.instances_path, filename) - if not os.path.exists(p): - raise exception.NotFound(_('%s not found') % p) - return os.remove(p) + tmp_file = os.path.join(FLAGS.instances_path, filename) + if not os.path.exists(tmp_file): + raise exception.NotFound(_('%s not found') % tmp_file) + + @exception.wrap_exception + def cleanup_shared_storage_test_file(self, context, filename): + """Removes existence of the tmpfile under FLAGS.instances_path. + + :param context: security context + :param filename: remove existence of FLAGS.instances_path/thisfile + + """ + + tmp_file = os.path.join(FLAGS.instances_path, filename) + os.remove(tmp_file) @exception.wrap_exception def update_available_resource(self, context): @@ -687,7 +698,7 @@ class ComputeManager(manager.Manager): raise exception.NotFound(msg % locals()) # If any volume is mounted, prepare here. - if len(instance_ref['volumes']) == 0: + if not instance_ref['volumes']: LOG.info(_("%s has no volume."), ec2_id) else: for v in instance_ref['volumes']: @@ -701,16 +712,16 @@ class ComputeManager(manager.Manager): # Retry operation is necessary because continuously request comes, # concorrent request occurs to iptables, then it complains. max_retry = FLAGS.live_migration_retry_count - for i in range(max_retry): + for cnt in range(max_retry): try: self.network_manager.setup_compute_network(context, instance_id) break - except exception.ProcessExecutionError, e: - if i == max_retry - 1: + except exception.ProcessExecutionError: + if cnt == max_retry - 1: raise else: - LOG.warn(_("setup_compute_network() failed %(i)d." + LOG.warn(_("setup_compute_network() failed %(cnt)d." "Retry up to %(max_retry)d for %(ec2_id)s.") % locals()) time.sleep(1) @@ -739,7 +750,7 @@ class ComputeManager(manager.Manager): try: # Checking volume node is working correctly when any volumes # are attached to instances. - if len(instance_ref['volumes']) != 0: + if instance_ref['volumes']: rpc.call(context, FLAGS.volume_topic, {"method": "check_for_export", @@ -751,7 +762,7 @@ class ComputeManager(manager.Manager): {"method": "pre_live_migration", "args": {'instance_id': instance_id}}) - except Exception, e: + except Exception: msg = _("Pre live migration for %(i_name)s failed at %(dest)s") LOG.error(msg % locals()) self.recover_live_migration(context, instance_ref) @@ -843,5 +854,5 @@ class ComputeManager(manager.Manager): 'state': power_state.RUNNING, 'host': host}) - for v in instance_ref['volumes']: - self.db.volume_update(ctxt, v['id'], {'status': 'in-use'}) + for volume in instance_ref['volumes']: + self.db.volume_update(ctxt, volume['id'], {'status': 'in-use'}) diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 73ce651da..4485ba39f 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -100,9 +100,9 @@ class Scheduler(object): 'migrating') # Changing volume state - for v in instance_ref['volumes']: + for volume_ref in instance_ref['volumes']: db.volume_update(context, - v['id'], + volume_ref['id'], {'status': 'migrating'}) # Return value is necessary to send request to src @@ -121,17 +121,16 @@ class Scheduler(object): # Checking instance is running. if (power_state.RUNNING != instance_ref['state'] or \ 'running' != instance_ref['state_description']): - msg = _('Instance(%s) is not running') ec2_id = instance_ref['hostname'] - raise exception.Invalid(msg % ec2_id) + raise exception.Invalid(_('Instance(%s) is not running') % ec2_id) # Checing volume node is running when any volumes are mounted # to the instance. if len(instance_ref['volumes']) != 0: services = db.service_get_all_by_topic(context, 'volume') if len(services) < 1 or not self.service_is_up(services[0]): - msg = _('volume node is not alive(time synchronize problem?)') - raise exception.Invalid(msg) + raise exception.Invalid(_("volume node is not alive" + "(time synchronize problem?)")) # Checking src host exists and compute node src = instance_ref['host'] @@ -139,8 +138,8 @@ class Scheduler(object): # Checking src host is alive. if not self.service_is_up(services[0]): - msg = _('%s is not alive(time synchronize problem?)') - raise exception.Invalid(msg % src) + raise exception.Invalid(_("%s is not alive(time " + "synchronize problem?)") % src) def _live_migration_dest_check(self, context, instance_ref, dest): """Live migration check routine (for destination host). @@ -157,8 +156,8 @@ class Scheduler(object): # Checking dest host is alive. if not self.service_is_up(dservice_ref): - msg = _('%s is not alive(time synchronize problem?)') - raise exception.Invalid(msg % dest) + raise exception.Invalid(_("%s is not alive(time " + "synchronize problem?)") % dest) # Checking whether The host where instance is running # and dest is not same. @@ -170,7 +169,9 @@ class Scheduler(object): % locals()) # Checking dst host still has enough capacities. - self.has_enough_resources(context, instance_ref, dest) + self.assert_compute_node_has_enough_resources(context, + instance_ref, + dest) def _live_migration_common_check(self, context, instance_ref, dest): """Live migration common check routine. @@ -202,18 +203,20 @@ class Scheduler(object): oservice_ref = oservice_refs[0]['compute_service'][0] # Checking hypervisor is same. - o = oservice_ref['hypervisor_type'] - d = dservice_ref['hypervisor_type'] - if o != d: + orig_hypervisor = oservice_ref['hypervisor_type'] + dest_hypervisor = dservice_ref['hypervisor_type'] + if orig_hypervisor != dest_hypervisor: raise exception.Invalid(_("Different hypervisor type" - "(%(o)s->%(d)s)')" % locals())) + "(%(orig_hypervisor)s->" + "%(dest_hypervisor)s)')" % locals())) # Checkng hypervisor version. - o = oservice_ref['hypervisor_version'] - d = dservice_ref['hypervisor_version'] - if o > d: - raise exception.Invalid(_('Older hypervisor version(%(o)s->%(d)s)') - % locals()) + orig_hypervisor = oservice_ref['hypervisor_version'] + dest_hypervisor = dservice_ref['hypervisor_version'] + if orig_hypervisor > dest_hypervisor: + raise exception.Invalid(_("Older hypervisor version" + "(%(orig_hypervisor)s->" + "%(dest_hypervisor)s)") % locals()) # Checking cpuinfo. try: @@ -222,14 +225,15 @@ class Scheduler(object): {"method": 'compare_cpu', "args": {'cpu_info': oservice_ref['cpu_info']}}) - except rpc.RemoteError, e: + except rpc.RemoteError: ec2_id = instance_ref['hostname'] src = instance_ref['host'] logging.exception(_("host %(dest)s is not compatible with " "original host %(src)s.") % locals()) raise - def has_enough_resources(self, context, instance_ref, dest): + def assert_compute_node_has_enough_resources(self, context, + instance_ref, dest): """Checks if destination host has enough resource for live migration. Currently, only memory checking has been done. @@ -276,22 +280,24 @@ class Scheduler(object): dst_t = db.queue_get_for(context, FLAGS.compute_topic, dest) src_t = db.queue_get_for(context, FLAGS.compute_topic, src) - # create tmpfile at dest host try: - filename = rpc.call(context, dst_t, {"method": 'mktmpfile'}) - except rpc.RemoteError, e: - msg = _("Cannot create tmpfile at %s to confirm shared storage.") - LOG.error(msg % FLAGS.instances_path) - raise + # create tmpfile at dest host + filename = rpc.call(context, dst_t, + {"method": 'create_shared_storage_test_file'}) - # make sure existence at src host. - try: + # make sure existence at src host. rpc.call(context, src_t, - {"method": 'confirm_tmpfile', + {"method": 'check_shared_storage_test_file', "args": {'filename': filename}}) - except (rpc.RemoteError, exception.NotFound), e: + except rpc.RemoteError: ipath = FLAGS.instances_path - logging.error(_("Cannot comfirm %(ipath)s at %(dest)s is " - "located at same shared storage.") % locals()) + logging.error(_("Cannot comfirm tmpfile at %(ipath)s is on " + "same shared storage between %(src)s " + "and %(dest)s.") % locals()) raise + + finally: + rpc.call(context, dst_t, + {"method": 'cleanup_shared_storage_test_file', + "args": {'filename': filename}}) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index cd5012fd5..a50d3ab20 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -98,24 +98,24 @@ class SchedulerManager(manager.Manager): # Getting usage resource information usage = {} instance_refs = db.instance_get_all_by_host(context, - compute_ref['host']) - if 0 == len(instance_refs): + compute_ref['host']) + if not instance_refs: return {'resource': resource, 'usage': usage} project_ids = [i['project_id'] for i in instance_refs] project_ids = list(set(project_ids)) - for i in project_ids: + for project_id in project_ids: vcpus = db.instance_get_vcpu_sum_by_host_and_project(context, host, - i) + project_id) mem = db.instance_get_memory_sum_by_host_and_project(context, host, - i) + project_id) hdd = db.instance_get_disk_sum_by_host_and_project(context, host, - i) - usage[i] = {'vcpus': int(vcpus), - 'memory_mb': int(mem), - 'local_gb': int(hdd)} + project_id) + usage[project_id] = {'vcpus': int(vcpus), + 'memory_mb': int(mem), + 'local_gb': int(hdd)} return {'resource': resource, 'usage': usage} diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index c4e4d148e..62db42b11 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -661,7 +661,6 @@ class SimpleDriverTestCase(test.TestCase): self.scheduler.live_migration(self.context, FLAGS.compute_topic, instance_id=instance_id, dest=i_ref['host']) - self.mox.UnsetStubs() i_ref = db.instance_get(self.context, instance_id) self.assertTrue(i_ref['state_description'] == 'migrating') @@ -824,10 +823,15 @@ class SimpleDriverTestCase(test.TestCase): topic = FLAGS.compute_topic driver.rpc.call(mox.IgnoreArg(), db.queue_get_for(self.context, topic, dest), - {"method": 'mktmpfile'}).AndReturn(fpath) + {"method": 'create_shared_storage_test_file'}).AndReturn(fpath) driver.rpc.call(mox.IgnoreArg(), db.queue_get_for(mox.IgnoreArg(), topic, i_ref['host']), - {"method": 'confirm_tmpfile', "args": {'filename': fpath}}) + {"method": 'check_shared_storage_test_file', + "args": {'filename': fpath}}) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(mox.IgnoreArg(), topic, dest), + {"method": 'cleanup_shared_storage_test_file', + "args": {'filename': fpath}}) self.mox.ReplayAll() try: @@ -838,7 +842,6 @@ class SimpleDriverTestCase(test.TestCase): c = (e.message.find('does not exist') >= 0) self.assertTrue(c) - self.mox.UnsetStubs() db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) @@ -867,7 +870,6 @@ class SimpleDriverTestCase(test.TestCase): c = (e.message.find(_('Different hypervisor type')) >= 0) self.assertTrue(c) - self.mox.UnsetStubs() db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) db.service_destroy(self.context, s_ref2['id']) @@ -898,7 +900,6 @@ class SimpleDriverTestCase(test.TestCase): c = (e.message.find(_('Older hypervisor version')) >= 0) self.assertTrue(c) - self.mox.UnsetStubs() db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) db.service_destroy(self.context, s_ref2['id']) @@ -934,7 +935,6 @@ class SimpleDriverTestCase(test.TestCase): c = (e.message.find(_("doesn't have compatibility to")) >= 0) self.assertTrue(c) - self.mox.UnsetStubs() db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) db.service_destroy(self.context, s_ref2['id']) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index f46b5950e..17b80c294 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -import libvirt import mox from xml.etree.ElementTree import fromstring as xml_to_tree @@ -60,6 +59,7 @@ class LibvirtConnTestCase(test.TestCase): admin=True) self.project = self.manager.create_project('fake', 'fake', 'fake') self.network = utils.import_object(FLAGS.network_manager) + self.context = context.get_admin_context() FLAGS.instances_path = '' self.call_libvirt_dependant_setup = False @@ -73,22 +73,52 @@ class LibvirtConnTestCase(test.TestCase): 'bridge': 'br101', 'instance_type': 'm1.small'} - def libvirt_dependant_setup(self): - """A setup method of LibvirtConnection dependent test.""" - # try to connect libvirt. if fail, skip test. - self.call_libvirt_dependant_setup = True - try: - libvirt.openReadOnly('qemu:///system') - except libvirt.libvirtError: - return - return libvirt_conn.get_connection(False) - - def libvirt_dependant_teardown(self): - """teardown method of LibvirtConnection dependent test.""" - if self.call_libvirt_dependant_setup: - libvirt_conn.libvirt = None - libvirt_conn.libxml2 = None - self.call_libvirt_dependant_setup = False + def create_fake_libvirt_mock(self, **kwargs): + """Defining mocks for LibvirtConnection(libvirt is not used).""" + + # A fake libvirt.virtConnect + class FakeLibvirtConnection(object): + def getVersion(self): + return 12003 + + def getType(self): + return 'qemu' + + def getCapabilities(self): + return 'qemu' + + def listDomainsID(self): + return [] + + def getCapabilitied(self): + return + + # A fake libvirt_conn.IptablesFirewallDriver + class FakeIptablesFirewallDriver(object): + def __init__(self, **kwargs): + pass + + # Creating mocks + fake = FakeLibvirtConnection() + fakeip = FakeIptablesFirewallDriver + # Customizing above fake if necessary + for key, val in kwargs.items(): + fake.__setattr__(key, val) + + # Inevitable mocks for libvirt_conn.LibvirtConnection + self.mox.StubOutWithMock(libvirt_conn.utils, 'import_class') + libvirt_conn.utils.import_class(mox.IgnoreArg()).AndReturn(fakeip) + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') + libvirt_conn.LibvirtConnection._conn = fake + + def create_service(self, **kwargs): + service_ref = {'host': kwargs.get('host', 'dummy'), + 'binary': 'nova-compute', + 'topic': 'compute', + 'report_count': 0, + 'availability_zone': 'zone'} + + return db.service_create(context.get_admin_context(), service_ref) def test_xml_and_uri_no_ramdisk_no_kernel(self): instance_data = dict(self.test_instance) @@ -244,306 +274,55 @@ class LibvirtConnTestCase(test.TestCase): self.assertEquals(uri, testuri) db.instance_destroy(user_context, instance_ref['id']) - def test_get_vcpu_used(self): - """Check if get_local_gb_total returns appropriate disk value.""" - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') - libvirt_conn.LibvirtConnection._conn.listDomainsID().AndReturn([1, 2]) - vdmock = self.mox.CreateMock(libvirt.virDomain) - self.mox.StubOutWithMock(vdmock, "vcpus") - vdmock.vcpus().AndReturn(['', [('dummycpu'), ('dummycpu')]]) - vdmock.vcpus().AndReturn(['', [('dummycpu'), ('dummycpu')]]) - arg = mox.IgnoreArg() - libvirt_conn.LibvirtConnection._conn.lookupByID(arg).AndReturn(vdmock) - libvirt_conn.LibvirtConnection._conn.lookupByID(arg).AndReturn(vdmock) - - self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) - self.assertTrue(conn.get_vcpu_used() == 4) - - def test_get_cpu_info_inappropreate_xml(self): - """Raise exception if given xml is inappropriate.""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - xml = """ - x86_64 - Nehalem - Intel - - - - - - - - - - - - - - - - - - """ - - self.mox.StubOutWithMock(conn._conn, 'getCapabilities') - conn._conn.getCapabilities().AndReturn(xml) - - self.mox.ReplayAll() - try: - conn.get_cpu_info() - except exception.Invalid, e: - c1 = (0 <= e.message.find('Invalid xml')) - self.assertTrue(c1) - - def test_get_cpu_info_inappropreate_xml2(self): - """Raise exception if given xml is inappropriate(topology tag).""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - xml = """ - x86_64 - Nehalem - Intel - - - - - - - - - - - - - - - - - """ - self.mox.StubOutWithMock(conn._conn, 'getCapabilities') - conn._conn.getCapabilities().AndReturn(xml) - - self.mox.ReplayAll() - try: - conn.get_cpu_info() - except exception.Invalid, e: - c1 = (0 <= e.message.find('Invalid xml: topology')) - self.assertTrue(c1) - def test_update_available_resource_works_correctly(self): """Confirm compute_service table is updated successfully.""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - host = 'dummy' - zone = 'dummyzone' - ctxt = context.get_admin_context() org_path = FLAGS.instances_path = '' FLAGS.instances_path = '.' - service_ref = db.service_create(ctxt, - {'host': host, - 'binary': 'nova-compute', - 'topic': 'compute', - 'report_count': 0, - 'availability_zone': zone}) - conn.update_available_resource(ctxt, host) + service_ref = self.create_service(host='dummy') + self.create_fake_libvirt_mock() + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, + 'get_cpu_info') + libvirt_conn.LibvirtConnection.get_cpu_info().AndReturn('cpuinfo') - service_ref = db.service_get(ctxt, service_ref['id']) - print service_ref['compute_service'] + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + conn.update_available_resource(self.context, 'dummy') + service_ref = db.service_get(self.context, service_ref['id']) compute_service = service_ref['compute_service'][0] + c1 = (compute_service['vcpus'] > 0) c2 = (compute_service['memory_mb'] > 0) c3 = (compute_service['local_gb'] > 0) - # vcpu_used is checked at test_get_vcpu_used. - c4 = (compute_service['memory_mb_used'] > 0) - c5 = (compute_service['local_gb_used'] > 0) - c6 = (len(compute_service['hypervisor_type']) > 0) - c7 = (compute_service['hypervisor_version'] > 0) + c4 = (compute_service['vcpus_used'] == 0) + c5 = (compute_service['memory_mb_used'] > 0) + c6 = (compute_service['local_gb_used'] > 0) + c7 = (len(compute_service['hypervisor_type']) > 0) + c8 = (compute_service['hypervisor_version'] > 0) - self.assertTrue(c1 and c2 and c3 and c4 and c5 and c6 and c7) + self.assertTrue(c1 and c2 and c3 and c4 and c5 and c6 and c7 and c8) - db.service_destroy(ctxt, service_ref['id']) + db.service_destroy(self.context, service_ref['id']) FLAGS.instances_path = org_path - def test_update_resource_info_raise_exception(self): + def test_update_resource_info_no_compute_record_found(self): """Raise exception if no recorde found on services table.""" - host = 'dummy' - org_path = FLAGS.instances_path = '' - FLAGS.instances_path = '.' - try: - conn = libvirt_conn.LibvirtConnection(False) - conn.update_available_resource(context.get_admin_context(), host) - except exception.Invalid, e: - msg = 'Cannot update compute manager specific info' - c1 = (0 <= e.message.find(msg)) - self.assertTrue(c1) - FLAGS.instances_path = org_path - - def test_compare_cpu_works_correctly(self): - """Calling libvirt.compute_cpu() and works correctly.""" - conn = self.libvirt_dependant_setup() - if not conn: - return - host = 'dummy' - zone = 'dummyzone' - ctxt = context.get_admin_context() org_path = FLAGS.instances_path = '' FLAGS.instances_path = '.' - - service_ref = db.service_create(ctxt, - {'host': host, - 'binary': 'nova-compute', - 'topic': 'compute', - 'report_count': 0, - 'availability_zone': zone}) - conn.update_available_resource(ctxt, host) - service_ref = db.service_get(ctxt, service_ref['id']) - ret = conn.compare_cpu(service_ref['compute_service'][0]['cpu_info']) - self.assertTrue(ret == None) - - db.service_destroy(ctxt, service_ref['id']) - FLAGS.instances_path = org_path - - def test_compare_cpu_no_compatibility(self): - """Libvirt.compare_cpu() return less than 0.(no compatibility).""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - t = {} - t['arch'] = 'x86' - t['model'] = 'model' - t['vendor'] = 'Intel' - t['topology'] = {'cores': "2", "threads": "1", "sockets": "4"} - t['features'] = ["tm"] - cpu_info = utils.dumps(t) - self.mox.StubOutWithMock(conn._conn, 'compareCPU') - conn._conn.compareCPU(mox.IgnoreArg(), 0).AndReturn(0) - - self.mox.ReplayAll() - self.assertRaises(exception.Invalid, conn.compare_cpu, cpu_info) - - def test_ensure_filtering_rules_for_instance_works_correctly(self): - """ensure_filtering_rules_for_instance() works successfully.""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - instance_ref = models.Instance() - instance_ref.__setitem__('id', 1) - fwdriver = conn.firewall_driver - - self.mox.StubOutWithMock(fwdriver, 'setup_basic_filtering') - fwdriver.setup_basic_filtering(instance_ref) - self.mox.StubOutWithMock(fwdriver, 'prepare_instance_filter') - fwdriver.prepare_instance_filter(instance_ref) - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') - n = 'nova-instance-%s' % instance_ref.name - conn._conn.nwfilterLookupByName(n) - - self.mox.ReplayAll() - conn.ensure_filtering_rules_for_instance(instance_ref) - - def test_ensure_filtering_rules_for_instance_timeout(self): - """ensure_filtering_fules_for_instance() finishes with timeout.""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - instance_ref = models.Instance() - instance_ref.__setitem__('id', 1) - fwdriver = conn.firewall_driver - - self.mox.StubOutWithMock(fwdriver, 'setup_basic_filtering') - fwdriver.setup_basic_filtering(instance_ref) - self.mox.StubOutWithMock(fwdriver, 'prepare_instance_filter') - fwdriver.prepare_instance_filter(instance_ref) - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') - n = 'nova-instance-%s' % instance_ref.name - for i in range(FLAGS.live_migration_retry_count): - conn._conn.nwfilterLookupByName(n).\ - AndRaise(libvirt.libvirtError('ERR')) - - self.mox.ReplayAll() - try: - conn.ensure_filtering_rules_for_instance(instance_ref) - except exception.Error, e: - c1 = (0 <= e.message.find('Timeout migrating for')) - self.assertTrue(c1) - - def test_live_migration_works_correctly(self): - """_live_migration() works as expected correctly.""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - class dummyCall(object): - f = None - - def start(self, interval=0, now=False): - pass - - i_ref = models.Instance() - i_ref.__setitem__('id', 1) - ctxt = context.get_admin_context() - - vdmock = self.mox.CreateMock(libvirt.virDomain) - self.mox.StubOutWithMock(vdmock, "migrateToURI") - vdmock.migrateToURI(FLAGS.live_migration_uri % 'dest', - mox.IgnoreArg(), - None, FLAGS.live_migration_bandwidth).\ - AndReturn(None) - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') - conn._conn.lookupByName(i_ref.name).AndReturn(vdmock) - self.mox.StubOutWithMock(libvirt_conn.utils, 'LoopingCall') - libvirt_conn.utils.LoopingCall(f=None).AndReturn(dummyCall()) + self.create_fake_libvirt_mock() self.mox.ReplayAll() - # Nothing to do with setting post_method/recover_method or not. - ret = conn._live_migration(ctxt, i_ref, 'dest', '', '') - self.assertTrue(ret == None) - - def test_live_migration_raises_exception(self): - """Confirms recover method is called when exceptions are raised.""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - i_ref = models.Instance() - i_ref.__setitem__('id', 1) - ctxt = context.get_admin_context() - - def dummy_recover_method(c, instance, host=None): - pass - - vdmock = self.mox.CreateMock(libvirt.virDomain) - self.mox.StubOutWithMock(vdmock, "migrateToURI") - vdmock.migrateToURI(FLAGS.live_migration_uri % 'dest', - mox.IgnoreArg(), - None, FLAGS.live_migration_bandwidth).\ - AndRaise(libvirt.libvirtError('ERR')) - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') - conn._conn.lookupByName(i_ref.name).AndReturn(vdmock) + conn = libvirt_conn.LibvirtConnection(False) + self.assertRaises(exception.Invalid, + conn.update_available_resource, + self.context, 'dummy') - self.mox.ReplayAll() - self.assertRaises(libvirt.libvirtError, - conn._live_migration, - ctxt, i_ref, 'dest', - '', dummy_recover_method) + FLAGS.instances_path = org_path def tearDown(self): self.manager.delete_project(self.project) self.manager.delete_user(self.user) super(LibvirtConnTestCase, self).tearDown() - self.libvirt_dependant_teardown() class IptablesFirewallTestCase(test.TestCase): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 75e4f0a53..70fdcc453 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -891,8 +891,8 @@ class LibvirtConnection(object): """ total = 0 - for i in self._conn.listDomainsID(): - dom = self._conn.lookupByID(i) + for dom_id in self._conn.listDomainsID(): + dom = self._conn.lookupByID(dom_id) total += len(dom.vcpus()[1]) return total @@ -1048,7 +1048,7 @@ class LibvirtConnection(object): 'cpu_info': self.get_cpu_info()} compute_service_ref = service_ref['compute_service'] - if len(compute_service_ref) == 0: + if not compute_service_ref: LOG.info(_('Compute_service record is created for %s ') % host) dic['service_id'] = service_ref['id'] db.compute_service_create(ctxt, dic) @@ -1124,7 +1124,7 @@ class LibvirtConnection(object): # wait for completion timeout_count = range(FLAGS.live_migration_retry_count) - while len(timeout_count) != 0: + while not timeout_count: try: filter_name = 'nova-instance-%s' % instance_ref.name self._conn.nwfilterLookupByName(filter_name) @@ -1198,7 +1198,7 @@ class LibvirtConnection(object): None, FLAGS.live_migration_bandwidth) - except Exception, e: + except Exception: recover_method(ctxt, instance_ref) raise diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 5dc9077b4..9dea35b35 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -175,5 +175,5 @@ class VolumeManager(manager.Manager): def check_for_export(self, context, instance_id): """Make sure whether volume is exported.""" instance_ref = self.db.instance_get(context, instance_id) - for v in instance_ref['volumes']: - self.driver.check_for_export(context, v['id']) + for volume in instance_ref['volumes']: + self.driver.check_for_export(context, volume['id']) -- cgit From 6797c5acc47fb5111ef821d6b074cb635692a9fb Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Thu, 3 Mar 2011 15:41:45 +0000 Subject: Add in multi-tenant support in openstack api. --- nova/api/openstack/__init__.py | 25 ++++++ nova/api/openstack/accounts.py | 73 ++++++++++++++++ nova/api/openstack/auth.py | 54 ++++++++++-- nova/api/openstack/backup_schedules.py | 6 +- nova/api/openstack/consoles.py | 10 +-- nova/api/openstack/flavors.py | 6 +- nova/api/openstack/images.py | 12 +-- nova/api/openstack/servers.py | 38 ++++---- nova/api/openstack/shared_ip_groups.py | 12 +-- nova/api/openstack/users.py | 93 ++++++++++++++++++++ nova/api/openstack/zones.py | 12 +-- nova/auth/novarc.template | 2 +- nova/db/sqlalchemy/api.py | 3 + nova/tests/api/openstack/fakes.py | 82 +++++++++++++++++- nova/tests/api/openstack/test_accounts.py | 123 ++++++++++++++++++++++++++ nova/tests/api/openstack/test_adminapi.py | 6 +- nova/tests/api/openstack/test_auth.py | 17 ++-- nova/tests/api/openstack/test_flavors.py | 4 +- nova/tests/api/openstack/test_images.py | 6 +- nova/tests/api/openstack/test_servers.py | 51 +++++------ nova/tests/api/openstack/test_users.py | 139 ++++++++++++++++++++++++++++++ nova/tests/api/openstack/test_zones.py | 12 +-- 22 files changed, 684 insertions(+), 102 deletions(-) create mode 100644 nova/api/openstack/accounts.py create mode 100644 nova/api/openstack/users.py create mode 100644 nova/tests/api/openstack/test_accounts.py create mode 100644 nova/tests/api/openstack/test_users.py (limited to 'nova') diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index b1b38ed2d..73d52192e 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -27,6 +27,7 @@ import webob.exc from nova import flags from nova import log as logging from nova import wsgi +from nova.api.openstack import accounts from nova.api.openstack import faults from nova.api.openstack import backup_schedules from nova.api.openstack import consoles @@ -34,6 +35,7 @@ from nova.api.openstack import flavors from nova.api.openstack import images from nova.api.openstack import servers from nova.api.openstack import shared_ip_groups +from nova.api.openstack import users from nova.api.openstack import zones @@ -71,6 +73,18 @@ class APIRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() + accounts_controller = accounts.Controller() + mapper.connect("account", "/{id}", + controller=accounts_controller, action="show", + conditions=dict(method=["GET"])) + if FLAGS.allow_admin_api: + mapper.connect("/{id}", + controller=accounts_controller, action="update", + conditions=dict(method=["PUT"])) + mapper.connect("/{id}", + controller=accounts_controller, action="delete", + conditions=dict(method=["DELETE"])) + server_members = {'action': 'POST'} if FLAGS.allow_admin_api: LOG.debug(_("Including admin operations in API.")) @@ -84,27 +98,38 @@ class APIRouter(wsgi.Router): server_members['inject_network_info'] = 'POST' mapper.resource("zone", "zones", controller=zones.Controller(), + path_prefix="{account_id}/", + collection={'detail': 'GET'}) + + mapper.resource("user", "users", controller=users.Controller(), + path_prefix="{account_id}/", collection={'detail': 'GET'}) mapper.resource("server", "servers", controller=servers.Controller(), collection={'detail': 'GET'}, + path_prefix="{account_id}/", member=server_members) mapper.resource("backup_schedule", "backup_schedule", controller=backup_schedules.Controller(), + path_prefix="{account_id}/servers/{server_id}/", parent_resource=dict(member_name='server', collection_name='servers')) mapper.resource("console", "consoles", controller=consoles.Controller(), + path_prefix="{account_id}/servers/{server_id}/", parent_resource=dict(member_name='server', collection_name='servers')) mapper.resource("image", "images", controller=images.Controller(), + path_prefix="{account_id}/", collection={'detail': 'GET'}) mapper.resource("flavor", "flavors", controller=flavors.Controller(), + path_prefix="{account_id}/", collection={'detail': 'GET'}) mapper.resource("shared_ip_group", "shared_ip_groups", + path_prefix="{account_id}/", collection={'detail': 'GET'}, controller=shared_ip_groups.Controller()) diff --git a/nova/api/openstack/accounts.py b/nova/api/openstack/accounts.py new file mode 100644 index 000000000..264fdab99 --- /dev/null +++ b/nova/api/openstack/accounts.py @@ -0,0 +1,73 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import common + +from nova import exception +from nova import flags +from nova import log as logging +from nova import wsgi + +from nova.auth import manager + +FLAGS = flags.FLAGS +LOG = logging.getLogger('nova.api.openstack') + + +def _translate_keys(account): + return dict(id=account.id, + name=account.name, + description=account.description, + manager=account.project_manager_id) + + +class Controller(wsgi.Controller): + + _serialization_metadata = { + 'application/xml': { + "attributes": { + "account": ["id", "name", "description", "manager"]}}} + + def __init__(self): + self.manager = manager.AuthManager() + + def _check_admin(self, context): + """ We cannot depend on the db layer to check for admin access + for the auth manager, so we do it here """ + if not context.is_admin: + raise exception.NotAuthorized("Not admin user.") + + def show(self, req, id): + """Return data about the given account id""" + account = self.manager.get_project(id) + return dict(account=_translate_keys(account)) + + def delete(self, req, id): + self._check_admin(req.environ['nova.context']) + self.manager.delete_project(id) + return {} + + def update(self, req, id): + """ This is really create or update. """ + self._check_admin(req.environ['nova.context']) + env = self._deserialize(req.body, req) + description = env['account'].get('description') + manager = env['account'].get('manager') + try: + account = self.manager.get_project(id) + self.manager.modify_project(id, manager, description) + except exception.NotFound: + account = self.manager.create_project(id, manager, description) + return dict(account=_translate_keys(account)) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 6011e6115..e77910fed 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -28,11 +28,13 @@ from nova import context from nova import db from nova import exception from nova import flags +from nova import log as logging from nova import manager from nova import utils from nova import wsgi from nova.api.openstack import faults +LOG = logging.getLogger('nova.api.openstack') FLAGS = flags.FLAGS @@ -50,14 +52,27 @@ class AuthMiddleware(wsgi.Middleware): def __call__(self, req): if not self.has_authentication(req): return self.authenticate(req) - user = self.get_user_by_authentication(req) + account_name = req.path_info_peek() if not user: return faults.Fault(webob.exc.HTTPUnauthorized()) - project = self.auth.get_project(FLAGS.default_project) - req.environ['nova.context'] = context.RequestContext(user, project) + if not account_name: + if self.auth.is_admin(user): + account_name = FLAGS.default_project + else: + return faults.Fault(webob.exc.HTTPUnauthorized()) + try: + account = self.auth.get_project(account_name) + except exception.NotFound: + return faults.Fault(webob.exc.HTTPUnauthorized()) + + if not self.auth.is_admin(user) and \ + not self.auth.is_project_member(user, account): + return faults.Fault(webob.exc.HTTPUnauthorized()) + + req.environ['nova.context'] = context.RequestContext(user, account) return self.application def has_authentication(self, req): @@ -70,6 +85,7 @@ class AuthMiddleware(wsgi.Middleware): # Unless the request is explicitly made against // don't # honor it path_info = req.path_info + account_name = None if len(path_info) > 1: return faults.Fault(webob.exc.HTTPUnauthorized()) @@ -79,7 +95,10 @@ class AuthMiddleware(wsgi.Middleware): except KeyError: return faults.Fault(webob.exc.HTTPUnauthorized()) - token, user = self._authorize_user(username, key, req) + if ':' in username: + account_name, username = username.rsplit(':', 1) + + token, user = self._authorize_user(username, account_name, key, req) if user and token: res = webob.Response() res.headers['X-Auth-Token'] = token.token_hash @@ -116,23 +135,44 @@ class AuthMiddleware(wsgi.Middleware): return self.auth.get_user(token.user_id) return None - def _authorize_user(self, username, key, req): + def _authorize_user(self, username, account_name, key, req): """Generates a new token and assigns it to a user. username - string + account_name - string key - string API key req - webob.Request object """ ctxt = context.get_admin_context() user = self.auth.get_user_from_access_key(key) + if account_name: + try: + account = self.auth.get_project(account_name) + except exception.NotFound: + return None, None + else: + # (dragondm) punt and try to determine account. + # this is something of a hack, but a user on 1 account is a + # common case, and is the way the current RS code works. + accounts = self.auth.get_projects(user=user) + if len(accounts) == 1: + account = accounts[0] + else: + #we can't tell what account they are logging in for. + return None, None + if user and user.name == username: token_hash = hashlib.sha1('%s%s%f' % (username, key, time.time())).hexdigest() token_dict = {} token_dict['token_hash'] = token_hash token_dict['cdn_management_url'] = '' - # Same as auth url, e.g. http://foo.org:8774/baz/v1.0 - token_dict['server_management_url'] = req.url + # auth url + project (account) id, e.g. + # http://foo.org:8774/baz/v1.0/myacct/ + os_url = '%s%s%s/' % (req.url, + '' if req.url.endswith('/') else '/', + account.id) + token_dict['server_management_url'] = os_url token_dict['storage_url'] = '' token_dict['user_id'] = user.id token = self.db.auth_token_create(ctxt, token_dict) diff --git a/nova/api/openstack/backup_schedules.py b/nova/api/openstack/backup_schedules.py index 7abb5f884..a4d5939df 100644 --- a/nova/api/openstack/backup_schedules.py +++ b/nova/api/openstack/backup_schedules.py @@ -40,15 +40,15 @@ class Controller(wsgi.Controller): def __init__(self): pass - def index(self, req, server_id): + def index(self, req, server_id, **kw): """ Returns the list of backup schedules for a given instance """ return _translate_keys({}) - def create(self, req, server_id): + def create(self, req, server_id, **kw): """ No actual update method required, since the existing API allows both create and update through a POST """ return faults.Fault(exc.HTTPNotImplemented()) - def delete(self, req, server_id, id): + def delete(self, req, server_id, id, **kw): """ Deletes an existing backup schedule """ return faults.Fault(exc.HTTPNotImplemented()) diff --git a/nova/api/openstack/consoles.py b/nova/api/openstack/consoles.py index 9ebdbe710..85b2a4140 100644 --- a/nova/api/openstack/consoles.py +++ b/nova/api/openstack/consoles.py @@ -55,7 +55,7 @@ class Controller(wsgi.Controller): self.console_api = console.API() super(Controller, self).__init__() - def index(self, req, server_id): + def index(self, req, server_id, **kw): """Returns a list of consoles for this instance""" consoles = self.console_api.get_consoles( req.environ['nova.context'], @@ -63,14 +63,14 @@ class Controller(wsgi.Controller): return dict(consoles=[_translate_keys(console) for console in consoles]) - def create(self, req, server_id): + def create(self, req, server_id, **kw): """Creates a new console""" #info = self._deserialize(req.body, req) self.console_api.create_console( req.environ['nova.context'], int(server_id)) - def show(self, req, server_id, id): + def show(self, req, server_id, id, **kw): """Shows in-depth information on a specific console""" try: console = self.console_api.get_console( @@ -81,11 +81,11 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return _translate_detail_keys(console) - def update(self, req, server_id, id): + def update(self, req, server_id, id, **kw): """You can't update a console""" raise faults.Fault(exc.HTTPNotImplemented()) - def delete(self, req, server_id, id): + def delete(self, req, server_id, id, **kw): """Deletes a console""" try: self.console_api.delete_console(req.environ['nova.context'], diff --git a/nova/api/openstack/flavors.py b/nova/api/openstack/flavors.py index f620d4107..79c3e1ab3 100644 --- a/nova/api/openstack/flavors.py +++ b/nova/api/openstack/flavors.py @@ -32,18 +32,18 @@ class Controller(wsgi.Controller): "attributes": { "flavor": ["id", "name", "ram", "disk"]}}} - def index(self, req): + def index(self, req, **kw): """Return all flavors in brief.""" return dict(flavors=[dict(id=flavor['id'], name=flavor['name']) for flavor in self.detail(req)['flavors']]) - def detail(self, req): + def detail(self, req, **kw): """Return all flavors in detail.""" items = [self.show(req, id)['flavor'] for id in self._all_ids()] items = common.limited(items, req) return dict(flavors=items) - def show(self, req, id): + def show(self, req, id, **kw): """Return data about the given flavor id.""" for name, val in instance_types.INSTANCE_TYPES.iteritems(): if val['flavorid'] == int(id): diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index cf85a496f..5bc5b9978 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -115,14 +115,14 @@ class Controller(wsgi.Controller): def __init__(self): self._service = utils.import_object(FLAGS.image_service) - def index(self, req): + def index(self, req, **kw): """Return all public images in brief""" items = self._service.index(req.environ['nova.context']) items = common.limited(items, req) items = [_filter_keys(item, ('id', 'name')) for item in items] return dict(images=items) - def detail(self, req): + def detail(self, req, **kw): """Return all public images in detail""" try: items = self._service.detail(req.environ['nova.context']) @@ -136,7 +136,7 @@ class Controller(wsgi.Controller): items = [_translate_status(item) for item in items] return dict(images=items) - def show(self, req, id): + def show(self, req, id, **kw): """Return data about the given image id""" image_id = common.get_image_id_from_image_hash(self._service, req.environ['nova.context'], id) @@ -145,11 +145,11 @@ class Controller(wsgi.Controller): _convert_image_id_to_hash(image) return dict(image=image) - def delete(self, req, id): + def delete(self, req, id, **kw): # Only public images are supported for now. raise faults.Fault(exc.HTTPNotFound()) - def create(self, req): + def create(self, req, **kw): context = req.environ['nova.context'] env = self._deserialize(req.body, req) instance_id = env["image"]["serverId"] @@ -160,7 +160,7 @@ class Controller(wsgi.Controller): return dict(image=image_meta) - def update(self, req, id): + def update(self, req, id, **kw): # Users may not modify public images, and that's all that # we support for now. raise faults.Fault(exc.HTTPNotFound()) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 69273ad7b..426de92be 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -105,11 +105,11 @@ class Controller(wsgi.Controller): self._image_service = utils.import_object(FLAGS.image_service) super(Controller, self).__init__() - def index(self, req): + def index(self, req, **kw): """ Returns a list of server names and ids for a given user """ return self._items(req, entity_maker=_translate_keys) - def detail(self, req): + def detail(self, req, **kw): """ Returns a list of server details for a given user """ return self._items(req, entity_maker=_translate_detail_keys) @@ -123,7 +123,7 @@ class Controller(wsgi.Controller): res = [entity_maker(inst)['server'] for inst in limited_list] return dict(servers=res) - def show(self, req, id): + def show(self, req, id, **kw): """ Returns server details by server id """ try: instance = self.compute_api.get(req.environ['nova.context'], id) @@ -131,7 +131,7 @@ class Controller(wsgi.Controller): except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) - def delete(self, req, id): + def delete(self, req, id, **kw): """ Destroys a server """ try: self.compute_api.delete(req.environ['nova.context'], id) @@ -139,7 +139,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def create(self, req): + def create(self, req, **kw): """ Creates a new server for a given user """ env = self._deserialize(req.body, req) if not env: @@ -180,7 +180,7 @@ class Controller(wsgi.Controller): onset_files=env.get('onset_files', [])) return _translate_keys(instances[0]) - def update(self, req, id): + def update(self, req, id, **kw): """ Updates the server name or password """ inst_dict = self._deserialize(req.body, req) if not inst_dict: @@ -202,7 +202,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPNoContent() - def action(self, req, id): + def action(self, req, id, **kw): """ Multi-purpose method used to reboot, rebuild, and resize a server """ input_dict = self._deserialize(req.body, req) @@ -219,7 +219,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def lock(self, req, id): + def lock(self, req, id, **kw): """ lock the instance with id admin only operation @@ -234,7 +234,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def unlock(self, req, id): + def unlock(self, req, id, **kw): """ unlock the instance with id admin only operation @@ -249,7 +249,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def get_lock(self, req, id): + def get_lock(self, req, id, **kw): """ return the boolean state of (instance with id)'s lock @@ -263,7 +263,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def reset_network(self, req, id): + def reset_network(self, req, id, **kw): """ Reset networking on an instance (admin only). @@ -277,7 +277,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def inject_network_info(self, req, id): + def inject_network_info(self, req, id, **kw): """ Inject network info for an instance (admin only). @@ -291,7 +291,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def pause(self, req, id): + def pause(self, req, id, **kw): """ Permit Admins to Pause the server. """ ctxt = req.environ['nova.context'] try: @@ -302,7 +302,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def unpause(self, req, id): + def unpause(self, req, id, **kw): """ Permit Admins to Unpause the server. """ ctxt = req.environ['nova.context'] try: @@ -313,7 +313,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def suspend(self, req, id): + def suspend(self, req, id, **kw): """permit admins to suspend the server""" context = req.environ['nova.context'] try: @@ -324,7 +324,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def resume(self, req, id): + def resume(self, req, id, **kw): """permit admins to resume the server from suspend""" context = req.environ['nova.context'] try: @@ -335,7 +335,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def get_ajax_console(self, req, id): + def get_ajax_console(self, req, id, **kw): """ Returns a url to an instance's ajaxterm console. """ try: self.compute_api.get_ajax_console(req.environ['nova.context'], @@ -344,12 +344,12 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def diagnostics(self, req, id): + def diagnostics(self, req, id, **kw): """Permit Admins to retrieve server diagnostics.""" ctxt = req.environ["nova.context"] return self.compute_api.get_diagnostics(ctxt, id) - def actions(self, req, id): + def actions(self, req, id, **kw): """Permit Admins to retrieve server actions.""" ctxt = req.environ["nova.context"] items = self.compute_api.get_actions(ctxt, id) diff --git a/nova/api/openstack/shared_ip_groups.py b/nova/api/openstack/shared_ip_groups.py index 5d78f9377..e3c917749 100644 --- a/nova/api/openstack/shared_ip_groups.py +++ b/nova/api/openstack/shared_ip_groups.py @@ -40,26 +40,26 @@ class Controller(wsgi.Controller): 'attributes': { 'sharedIpGroup': []}}} - def index(self, req): + def index(self, req, **kw): """ Returns a list of Shared IP Groups for the user """ return dict(sharedIpGroups=[]) - def show(self, req, id): + def show(self, req, id, **kw): """ Shows in-depth information on a specific Shared IP Group """ return _translate_keys({}) - def update(self, req, id): + def update(self, req, id, **kw): """ You can't update a Shared IP Group """ raise faults.Fault(exc.HTTPNotImplemented()) - def delete(self, req, id): + def delete(self, req, id, **kw): """ Deletes a Shared IP Group """ raise faults.Fault(exc.HTTPNotImplemented()) - def detail(self, req): + def detail(self, req, **kw): """ Returns a complete list of Shared IP Groups """ return _translate_detail_keys({}) - def create(self, req): + def create(self, req, **kw): """ Creates a new Shared IP group """ raise faults.Fault(exc.HTTPNotImplemented()) diff --git a/nova/api/openstack/users.py b/nova/api/openstack/users.py new file mode 100644 index 000000000..c0b7544f9 --- /dev/null +++ b/nova/api/openstack/users.py @@ -0,0 +1,93 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import common + +from nova import exception +from nova import flags +from nova import log as logging +from nova import wsgi + +from nova.auth import manager + +FLAGS = flags.FLAGS +LOG = logging.getLogger('nova.api.openstack') + + +def _translate_keys(user): + return dict(id=user.id, + name=user.name, + access=user.access, + secret=user.secret, + admin=user.admin) + + +class Controller(wsgi.Controller): + + _serialization_metadata = { + 'application/xml': { + "attributes": { + "user": ["id", "name", "access", "secret", "admin"]}}} + + def __init__(self): + self.manager = manager.AuthManager() + + def _check_admin(self, context): + """ We cannot depend on the db layer to check for admin access + for the auth manager, so we do it here """ + if not context.is_admin: + raise exception.NotAuthorized("Not admin user") + + def index(self, req, **kw): + """Return all users in brief""" + users = self.manager.get_users() + users = common.limited(users, req) + users = [_translate_keys(user) for user in users] + return dict(users=users) + + def detail(self, req, **kw): + """Return all users in detail""" + return self.index(req) + + def show(self, req, id, **kw): + """Return data about the given user id""" + user = self.manager.get_user(id) + return dict(user=_translate_keys(user)) + + def delete(self, req, id, **kw): + self._check_admin(req.environ['nova.context']) + self.manager.delete_user(id) + return {} + + def create(self, req, **kw): + self._check_admin(req.environ['nova.context']) + env = self._deserialize(req.body, req) + is_admin = env['user'].get('admin') in ('T', 'True', True) + name = env['user'].get('name') + access = env['user'].get('access') + secret = env['user'].get('secret') + user = self.manager.create_user(name, access, secret, is_admin) + return dict(user=_translate_keys(user)) + + def update(self, req, id, **kw): + self._check_admin(req.environ['nova.context']) + env = self._deserialize(req.body, req) + is_admin = env['user'].get('admin') + if is_admin is not None: + is_admin = is_admin in ('T', 'True', True) + access = env['user'].get('access') + secret = env['user'].get('secret') + self.manager.modify_user(id, access, secret, is_admin) + return dict(user=_translate_keys(self.manager.get_user(id))) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index d5206da20..30bf2b67b 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -43,35 +43,35 @@ class Controller(wsgi.Controller): "attributes": { "zone": ["id", "api_url"]}}} - def index(self, req): + def index(self, req, **kw): """Return all zones in brief""" items = db.zone_get_all(req.environ['nova.context']) items = common.limited(items, req) items = [_scrub_zone(item) for item in items] return dict(zones=items) - def detail(self, req): + def detail(self, req, **kw): """Return all zones in detail""" return self.index(req) - def show(self, req, id): + def show(self, req, id, **kw): """Return data about the given zone id""" zone_id = int(id) zone = db.zone_get(req.environ['nova.context'], zone_id) return dict(zone=_scrub_zone(zone)) - def delete(self, req, id): + def delete(self, req, id, **kw): zone_id = int(id) db.zone_delete(req.environ['nova.context'], zone_id) return {} - def create(self, req): + def create(self, req, **kw): context = req.environ['nova.context'] env = self._deserialize(req.body, req) zone = db.zone_create(context, env["zone"]) return dict(zone=_scrub_zone(zone)) - def update(self, req, id): + def update(self, req, id, **kw): context = req.environ['nova.context'] env = self._deserialize(req.body, req) zone_id = int(id) diff --git a/nova/auth/novarc.template b/nova/auth/novarc.template index cda2ecc28..1c917ad44 100644 --- a/nova/auth/novarc.template +++ b/nova/auth/novarc.template @@ -11,5 +11,5 @@ export EUCALYPTUS_CERT=${NOVA_CERT} # euca-bundle-image seems to require this se alias ec2-bundle-image="ec2-bundle-image --cert ${EC2_CERT} --privatekey ${EC2_PRIVATE_KEY} --user 42 --ec2cert ${NOVA_CERT}" alias ec2-upload-bundle="ec2-upload-bundle -a ${EC2_ACCESS_KEY} -s ${EC2_SECRET_KEY} --url ${S3_URL} --ec2cert ${NOVA_CERT}" export NOVA_API_KEY="%(access)s" -export NOVA_USERNAME="%(user)s" +export NOVA_USERNAME="%(project)s:%(user)s" export NOVA_URL="%(os)s" diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 6df2a8843..e311f310a 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1861,8 +1861,11 @@ def project_get_by_user(context, user_id): session = get_session() user = session.query(models.User).\ filter_by(deleted=can_read_deleted(context)).\ + filter_by(id=user_id).\ options(joinedload_all('projects')).\ first() + if not user: + raise exception.NotFound(_('Invalid user_id %s') % user_id) return user.projects diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 49ce8c1b5..03b26e29a 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -26,7 +26,6 @@ from paste import urlmap from glance import client as glance_client -from nova import auth from nova import context from nova import exception as exc from nova import flags @@ -35,6 +34,7 @@ import nova.api.openstack.auth from nova.api import openstack from nova.api.openstack import auth from nova.api.openstack import ratelimiting +from nova.auth.manager import User, Project from nova.image import glance from nova.image import local from nova.image import service @@ -227,19 +227,97 @@ class FakeAuthDatabase(object): class FakeAuthManager(object): auth_data = {} + projects = {} + + @classmethod + def clear_fakes(cls): + cls.auth_data = {} + cls.projects = {} + + @classmethod + def reset_fake_data(cls): + cls.auth_data = dict(acc1=User('guy1', 'guy1', 'acc1', + 'fortytwo!', False)) + cls.projects = dict(testacct=Project('testacct', + 'testacct', + 'guy1', + 'test', + [])) def add_user(self, key, user): FakeAuthManager.auth_data[key] = user + def get_users(self): + return FakeAuthManager.auth_data.values() + def get_user(self, uid): for k, v in FakeAuthManager.auth_data.iteritems(): if v.id == uid: return v return None - def get_project(self, pid): + def delete_user(self, uid): + for k, v in FakeAuthManager.auth_data.items(): + if v.id == uid: + del FakeAuthManager.auth_data[k] return None + def create_user(self, name, access=None, secret=None, admin=False): + u = User(name, name, access, secret, admin) + FakeAuthManager.auth_data[access] = u + return u + + def modify_user(self, user_id, access=None, secret=None, admin=None): + user = None + for k, v in FakeAuthManager.auth_data.iteritems(): + if v.id == user_id: + user = v + if user: + user.access = access + user.secret = secret + if admin is not None: + user.admin = admin + + def is_admin(self, user): + return user.admin + + def is_project_member(self, user, project): + return ((user.id in project.member_ids) or + (user.id == project.project_manager_id)) + + def create_project(self, name, manager_user, description=None, + member_users=None): + member_ids = [User.safe_id(m) for m in member_users] \ + if member_users else [] + p = Project(name, name, User.safe_id(manager_user), + description, member_ids) + FakeAuthManager.projects[name] = p + return p + + def delete_project(self, pid): + if pid in FakeAuthManager.projects: + del FakeAuthManager.projects[pid] + + def modify_project(self, project, manager_user=None, description=None): + p = FakeAuthManager.projects.get(project) + p.project_manager_id = User.safe_id(manager_user) + p.description = description + + def get_project(self, pid): + p = FakeAuthManager.projects.get(pid) + if p: + return p + else: + raise exc.NotFound + + def get_projects(self, user=None): + if not user: + return FakeAuthManager.projects.values() + else: + return [p for p in FakeAuthManager.projects.values() + if (user.id in p.member_ids) or + (user.id == p.project_manager_id)] + def get_user_from_access_key(self, key): return FakeAuthManager.auth_data.get(key, None) diff --git a/nova/tests/api/openstack/test_accounts.py b/nova/tests/api/openstack/test_accounts.py new file mode 100644 index 000000000..b2e89824a --- /dev/null +++ b/nova/tests/api/openstack/test_accounts.py @@ -0,0 +1,123 @@ +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import stubout +import webob +import json + +import nova.api +import nova.api.openstack.auth +from nova import context +from nova import flags +from nova import test +from nova.auth.manager import User +from nova.tests.api.openstack import fakes + + +FLAGS = flags.FLAGS +FLAGS.verbose = True + + +def fake_init(self): + self.manager = fakes.FakeAuthManager() + + +def fake_admin_check(self, req): + return True + + +class AccountsTest(test.TestCase): + def setUp(self): + super(AccountsTest, self).setUp() + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(nova.api.openstack.accounts.Controller, '__init__', + fake_init) + self.stubs.Set(nova.api.openstack.accounts.Controller, '_check_admin', + fake_admin_check) + fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.projects = {} + fakes.FakeAuthDatabase.data = {} + fakes.stub_out_networking(self.stubs) + fakes.stub_out_rate_limiting(self.stubs) + fakes.stub_out_auth(self.stubs) + + self.allow_admin = FLAGS.allow_admin_api + FLAGS.allow_admin_api = True + fakemgr = fakes.FakeAuthManager() + joeuser = User('guy1', 'guy1', 'acc1', 'fortytwo!', False) + superuser = User('guy2', 'guy2', 'acc2', 'swordfish', True) + fakemgr.add_user(joeuser.access, joeuser) + fakemgr.add_user(superuser.access, superuser) + fakemgr.create_project('test1', joeuser) + fakemgr.create_project('test2', superuser) + + def tearDown(self): + self.stubs.UnsetAll() + FLAGS.allow_admin_api = self.allow_admin + super(AccountsTest, self).tearDown() + + def test_get_account(self): + req = webob.Request.blank('/v1.0/test1') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res_dict['account']['id'], 'test1') + self.assertEqual(res_dict['account']['name'], 'test1') + self.assertEqual(res_dict['account']['manager'], 'guy1') + self.assertEqual(res.status_int, 200) + + def test_account_delete(self): + req = webob.Request.blank('/v1.0/test1') + req.method = 'DELETE' + res = req.get_response(fakes.wsgi_app()) + self.assertTrue('test1' not in fakes.FakeAuthManager.projects) + self.assertEqual(res.status_int, 200) + + def test_account_create(self): + body = dict(account=dict(description='test account', + manager='guy1')) + req = webob.Request.blank('/v1.0/newacct') + req.method = 'PUT' + req.body = json.dumps(body) + + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(res_dict['account']['id'], 'newacct') + self.assertEqual(res_dict['account']['name'], 'newacct') + self.assertEqual(res_dict['account']['description'], 'test account') + self.assertEqual(res_dict['account']['manager'], 'guy1') + self.assertTrue('newacct' in + fakes.FakeAuthManager.projects) + self.assertEqual(len(fakes.FakeAuthManager.projects.values()), 3) + + def test_account_update(self): + body = dict(account=dict(description='test account', + manager='guy2')) + req = webob.Request.blank('/v1.0/test1') + req.method = 'PUT' + req.body = json.dumps(body) + + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(res_dict['account']['id'], 'test1') + self.assertEqual(res_dict['account']['name'], 'test1') + self.assertEqual(res_dict['account']['description'], 'test account') + self.assertEqual(res_dict['account']['manager'], 'guy2') + self.assertEqual(len(fakes.FakeAuthManager.projects.values()), 2) diff --git a/nova/tests/api/openstack/test_adminapi.py b/nova/tests/api/openstack/test_adminapi.py index dfce1b127..7cb9e8450 100644 --- a/nova/tests/api/openstack/test_adminapi.py +++ b/nova/tests/api/openstack/test_adminapi.py @@ -35,7 +35,7 @@ class AdminAPITest(test.TestCase): def setUp(self): super(AdminAPITest, self).setUp() self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) @@ -50,7 +50,7 @@ class AdminAPITest(test.TestCase): def test_admin_enabled(self): FLAGS.allow_admin_api = True # We should still be able to access public operations. - req = webob.Request.blank('/v1.0/flavors') + req = webob.Request.blank('/v1.0/testacct/flavors') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) # TODO: Confirm admin operations are available. @@ -58,7 +58,7 @@ class AdminAPITest(test.TestCase): def test_admin_disabled(self): FLAGS.allow_admin_api = False # We should still be able to access public operations. - req = webob.Request.blank('/v1.0/flavors') + req = webob.Request.blank('/v1.0/testacct/flavors') res = req.get_response(fakes.wsgi_app()) # TODO: Confirm admin operations are unavailable. self.assertEqual(res.status_int, 200) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index ff8d42a14..8268a6fb9 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -51,7 +51,9 @@ class Test(test.TestCase): def test_authorize_user(self): f = fakes.FakeAuthManager() - f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) + u = nova.auth.manager.User(1, 'herp', None, None, None) + f.add_user('derp', u) + f.create_project('test', u) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' @@ -65,7 +67,9 @@ class Test(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) + u = nova.auth.manager.User(1, 'herp', None, None, None) + f.add_user('derp', u) + f.create_project('test', u) req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) req.headers['X-Auth-User'] = 'herp' @@ -74,7 +78,7 @@ class Test(test.TestCase): self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) self.assertEqual(result.headers['X-Server-Management-Url'], - "http://foo/v1.0/") + "http://foo/v1.0/test/") self.assertEqual(result.headers['X-CDN-Management-Url'], "") self.assertEqual(result.headers['X-Storage-Url'], "") @@ -82,7 +86,7 @@ class Test(test.TestCase): token = result.headers['X-Auth-Token'] self.stubs.Set(nova.api.openstack, 'APIRouter', fakes.FakeRouter) - req = webob.Request.blank('/v1.0/fake') + req = webob.Request.blank('/v1.0/test/fake') req.headers['X-Auth-Token'] = token result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '200 OK') @@ -176,6 +180,9 @@ class TestLimiter(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() + u = nova.auth.manager.User(1, 'herp', None, None, None) + f.add_user('derp', u) + f.create_project('test', u) f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/') @@ -187,7 +194,7 @@ class TestLimiter(test.TestCase): token = result.headers['X-Auth-Token'] self.stubs.Set(nova.api.openstack, 'APIRouter', fakes.FakeRouter) - req = webob.Request.blank('/v1.0/fake') + req = webob.Request.blank('/v1.0/test/fake') req.method = 'POST' req.headers['X-Auth-Token'] = token result = req.get_response(fakes.wsgi_app()) diff --git a/nova/tests/api/openstack/test_flavors.py b/nova/tests/api/openstack/test_flavors.py index 761265965..370dc007c 100644 --- a/nova/tests/api/openstack/test_flavors.py +++ b/nova/tests/api/openstack/test_flavors.py @@ -28,7 +28,7 @@ class FlavorsTest(test.TestCase): def setUp(self): super(FlavorsTest, self).setUp() self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) @@ -39,7 +39,7 @@ class FlavorsTest(test.TestCase): super(FlavorsTest, self).tearDown() def test_get_flavor_list(self): - req = webob.Request.blank('/v1.0/flavors') + req = webob.Request.blank('/v1.0/testacct/flavors') res = req.get_response(fakes.wsgi_app()) def test_get_flavor_by_id(self): diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index e232bc3d5..819ca001e 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -202,7 +202,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): self.orig_image_service = FLAGS.image_service FLAGS.image_service = 'nova.image.glance.GlanceImageService' self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) @@ -216,7 +216,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): super(ImageControllerWithGlanceServiceTest, self).tearDown() def test_get_image_index(self): - req = webob.Request.blank('/v1.0/images') + req = webob.Request.blank('/v1.0/testacct/images') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -228,7 +228,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): "image %s not in fixture index!" % str(image)) def test_get_image_details(self): - req = webob.Request.blank('/v1.0/images/detail') + req = webob.Request.blank('/v1.0/testacct/images/detail') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 78beb7df9..d592e06b0 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -118,7 +118,7 @@ class ServersTest(test.TestCase): def setUp(self): super(ServersTest, self).setUp() self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) @@ -150,7 +150,7 @@ class ServersTest(test.TestCase): super(ServersTest, self).tearDown() def test_get_server_by_id(self): - req = webob.Request.blank('/v1.0/servers/1') + req = webob.Request.blank('/v1.0/testacct/servers/1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(res_dict['server']['id'], '1') @@ -161,7 +161,7 @@ class ServersTest(test.TestCase): public = ["1.2.3.4"] new_return_server = return_server_with_addresses(private, public) self.stubs.Set(nova.db.api, 'instance_get', new_return_server) - req = webob.Request.blank('/v1.0/servers/1') + req = webob.Request.blank('/v1.0/testacct/servers/1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(res_dict['server']['id'], '1') @@ -173,7 +173,7 @@ class ServersTest(test.TestCase): self.assertEqual(addresses["private"][0], private) def test_get_server_list(self): - req = webob.Request.blank('/v1.0/servers') + req = webob.Request.blank('/v1.0/testacct/servers') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -224,7 +224,7 @@ class ServersTest(test.TestCase): name='server_test', imageId=2, flavorId=2, metadata={'hello': 'world', 'open': 'stack'}, personality={})) - req = webob.Request.blank('/v1.0/servers') + req = webob.Request.blank('/v1.0/testacct/servers') req.method = 'POST' req.body = json.dumps(body) @@ -233,7 +233,7 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) def test_update_no_body(self): - req = webob.Request.blank('/v1.0/servers/1') + req = webob.Request.blank('/v1.0/testacct/servers/1') req.method = 'PUT' res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 422) @@ -251,7 +251,7 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'instance_update', server_update) - req = webob.Request.blank('/v1.0/servers/1') + req = webob.Request.blank('/v1.0/testacct/servers/1') req.method = 'PUT' req.body = self.body req.get_response(fakes.wsgi_app()) @@ -267,30 +267,30 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'instance_update', server_update) - req = webob.Request.blank('/v1.0/servers/1') + req = webob.Request.blank('/v1.0/testacct/servers/1') req.method = 'PUT' req.body = self.body req.get_response(fakes.wsgi_app()) def test_create_backup_schedules(self): - req = webob.Request.blank('/v1.0/servers/1/backup_schedules') + req = webob.Request.blank('/v1.0/testacct/servers/1/backup_schedules') req.method = 'POST' res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_delete_backup_schedules(self): - req = webob.Request.blank('/v1.0/servers/1/backup_schedules') + req = webob.Request.blank('/v1.0/testacct/servers/1/backup_schedules') req.method = 'DELETE' res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_get_server_backup_schedules(self): - req = webob.Request.blank('/v1.0/servers/1/backup_schedules') + req = webob.Request.blank('/v1.0/testacct/servers/1/backup_schedules') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_get_all_server_details(self): - req = webob.Request.blank('/v1.0/servers/detail') + req = webob.Request.blank('/v1.0/testacct/servers/detail') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -321,7 +321,7 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'instance_get_all_by_user', return_servers_with_host) - req = webob.Request.blank('/v1.0/servers/detail') + req = webob.Request.blank('/v1.0/testacct/servers/detail') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -341,7 +341,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/pause') + req = webob.Request.blank('/v1.0/testacct/servers/1/pause') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -353,7 +353,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/unpause') + req = webob.Request.blank('/v1.0/testacct/servers/1/unpause') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -365,7 +365,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/suspend') + req = webob.Request.blank('/v1.0/testacct/servers/1/suspend') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -377,7 +377,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/resume') + req = webob.Request.blank('/v1.0/testacct/servers/1/resume') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -389,7 +389,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/reset_network') + req = webob.Request.blank('/v1.0/testacct/servers/1/reset_network') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -401,7 +401,8 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/inject_network_info') + req = webob.Request.blank( + '/v1.0/testacct/servers/1/inject_network_info') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -409,13 +410,13 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 202) def test_server_diagnostics(self): - req = webob.Request.blank("/v1.0/servers/1/diagnostics") + req = webob.Request.blank("/v1.0/testacct/servers/1/diagnostics") req.method = "GET" res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 404) def test_server_actions(self): - req = webob.Request.blank("/v1.0/servers/1/actions") + req = webob.Request.blank("/v1.0/testacct/servers/1/actions") req.method = "GET" res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 404) @@ -424,7 +425,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/action') + req = webob.Request.blank('/v1.0/testacct/servers/1/action') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -434,7 +435,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/action') + req = webob.Request.blank('/v1.0/testacct/servers/1/action') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -444,14 +445,14 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/servers/1/action') + req = webob.Request.blank('/v1.0/testacct/servers/1/action') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) res = req.get_response(fakes.wsgi_app()) def test_delete_server_instance(self): - req = webob.Request.blank('/v1.0/servers/1') + req = webob.Request.blank('/v1.0/testacct/servers/1') req.method = 'DELETE' self.server_delete_called = False diff --git a/nova/tests/api/openstack/test_users.py b/nova/tests/api/openstack/test_users.py new file mode 100644 index 000000000..bd32254cd --- /dev/null +++ b/nova/tests/api/openstack/test_users.py @@ -0,0 +1,139 @@ +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import stubout +import webob +import json + +import nova.api +import nova.api.openstack.auth +from nova import context +from nova import flags +from nova import test +from nova.auth.manager import User, Project +from nova.tests.api.openstack import fakes + + +FLAGS = flags.FLAGS +FLAGS.verbose = True + + +def fake_init(self): + self.manager = fakes.FakeAuthManager() + + +def fake_admin_check(self, req): + return True + + +class UsersTest(test.TestCase): + def setUp(self): + super(UsersTest, self).setUp() + self.stubs = stubout.StubOutForTesting() + self.stubs.Set(nova.api.openstack.users.Controller, '__init__', + fake_init) + self.stubs.Set(nova.api.openstack.users.Controller, '_check_admin', + fake_admin_check) + fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.projects = dict(testacct=Project('testacct', + 'testacct', + 'guy1', + 'test', + [])) + fakes.FakeAuthDatabase.data = {} + fakes.stub_out_networking(self.stubs) + fakes.stub_out_rate_limiting(self.stubs) + fakes.stub_out_auth(self.stubs) + + self.allow_admin = FLAGS.allow_admin_api + FLAGS.allow_admin_api = True + fakemgr = fakes.FakeAuthManager() + fakemgr.add_user('acc1', User('guy1', 'guy1', 'acc1', + 'fortytwo!', False)) + fakemgr.add_user('acc2', User('guy2', 'guy2', 'acc2', + 'swordfish', True)) + + def tearDown(self): + self.stubs.UnsetAll() + FLAGS.allow_admin_api = self.allow_admin + super(UsersTest, self).tearDown() + + def test_get_user_list(self): + req = webob.Request.blank('/v1.0/testacct/users') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(len(res_dict['users']), 2) + + def test_get_user_by_id(self): + req = webob.Request.blank('/v1.0/testacct/users/guy2') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res_dict['user']['id'], 'guy2') + self.assertEqual(res_dict['user']['name'], 'guy2') + self.assertEqual(res_dict['user']['secret'], 'swordfish') + self.assertEqual(res_dict['user']['admin'], True) + self.assertEqual(res.status_int, 200) + + def test_user_delete(self): + req = webob.Request.blank('/v1.0/testacct/users/guy1') + req.method = 'DELETE' + res = req.get_response(fakes.wsgi_app()) + self.assertTrue('guy1' not in [u.id for u in + fakes.FakeAuthManager.auth_data.values()]) + self.assertEqual(res.status_int, 200) + + def test_user_create(self): + body = dict(user=dict(name='test_guy', + access='acc3', + secret='invasionIsInNormandy', + admin=True)) + req = webob.Request.blank('/v1.0/testacct/users') + req.method = 'POST' + req.body = json.dumps(body) + + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(res_dict['user']['id'], 'test_guy') + self.assertEqual(res_dict['user']['name'], 'test_guy') + self.assertEqual(res_dict['user']['access'], 'acc3') + self.assertEqual(res_dict['user']['secret'], 'invasionIsInNormandy') + self.assertEqual(res_dict['user']['admin'], True) + self.assertTrue('test_guy' in [u.id for u in + fakes.FakeAuthManager.auth_data.values()]) + self.assertEqual(len(fakes.FakeAuthManager.auth_data.values()), 3) + + def test_user_update(self): + body = dict(user=dict(name='guy2', + access='acc2', + secret='invasionIsInNormandy')) + req = webob.Request.blank('/v1.0/testacct/users/guy2') + req.method = 'PUT' + req.body = json.dumps(body) + + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(res.status_int, 200) + self.assertEqual(res_dict['user']['id'], 'guy2') + self.assertEqual(res_dict['user']['name'], 'guy2') + self.assertEqual(res_dict['user']['access'], 'acc2') + self.assertEqual(res_dict['user']['secret'], 'invasionIsInNormandy') + self.assertEqual(res_dict['user']['admin'], True) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 555b206b9..51f13af48 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -64,7 +64,7 @@ class ZonesTest(test.TestCase): def setUp(self): super(ZonesTest, self).setUp() self.stubs = stubout.StubOutForTesting() - fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthManager.reset_fake_data() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) @@ -85,7 +85,7 @@ class ZonesTest(test.TestCase): super(ZonesTest, self).tearDown() def test_get_zone_list(self): - req = webob.Request.blank('/v1.0/zones') + req = webob.Request.blank('/v1.0/testacct/zones') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -93,7 +93,7 @@ class ZonesTest(test.TestCase): self.assertEqual(len(res_dict['zones']), 2) def test_get_zone_by_id(self): - req = webob.Request.blank('/v1.0/zones/1') + req = webob.Request.blank('/v1.0/testacct/zones/1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -103,7 +103,7 @@ class ZonesTest(test.TestCase): self.assertEqual(res.status_int, 200) def test_zone_delete(self): - req = webob.Request.blank('/v1.0/zones/1') + req = webob.Request.blank('/v1.0/testacct/zones/1') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) @@ -111,7 +111,7 @@ class ZonesTest(test.TestCase): def test_zone_create(self): body = dict(zone=dict(api_url='http://blah.zoo', username='fred', password='fubar')) - req = webob.Request.blank('/v1.0/zones') + req = webob.Request.blank('/v1.0/testacct/zones') req.method = 'POST' req.body = json.dumps(body) @@ -125,7 +125,7 @@ class ZonesTest(test.TestCase): def test_zone_update(self): body = dict(zone=dict(username='zeb', password='sneaky')) - req = webob.Request.blank('/v1.0/zones/1') + req = webob.Request.blank('/v1.0/testacct/zones/1') req.method = 'PUT' req.body = json.dumps(body) -- cgit From a62e603e8b1cedd89ca0c71f1cdc928d19c68a4d Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 3 Mar 2011 11:04:33 -0500 Subject: adding wsgi.Request class to add custom best_match; adding new class to wsgify decorators; replacing all references to webob.Request in non-test code to wsgi.Request --- nova/api/direct.py | 4 ++-- nova/api/ec2/__init__.py | 14 +++++------ nova/api/ec2/metadatarequesthandler.py | 2 +- nova/api/openstack/__init__.py | 4 ++-- nova/api/openstack/auth.py | 4 ++-- nova/api/openstack/common.py | 2 +- nova/api/openstack/faults.py | 2 +- nova/api/openstack/ratelimiting/__init__.py | 4 ++-- nova/wsgi.py | 37 ++++++++++++++++++++++------- 9 files changed, 47 insertions(+), 26 deletions(-) (limited to 'nova') diff --git a/nova/api/direct.py b/nova/api/direct.py index 208b6d086..cd237f649 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -187,7 +187,7 @@ class ServiceWrapper(wsgi.Controller): def __init__(self, service_handle): self.service_handle = service_handle - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): arg_dict = req.environ['wsgiorg.routing_args'][1] action = arg_dict['action'] @@ -218,7 +218,7 @@ class Proxy(object): self.prefix = prefix def __do_request(self, path, context, **kwargs): - req = webob.Request.blank(path) + req = wsgi.Request.blank(path) req.method = 'POST' req.body = urllib.urlencode({'json': utils.dumps(kwargs)}) req.environ['openstack.context'] = context diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 5adc2c075..58b1ecf03 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -53,7 +53,7 @@ flags.DEFINE_list('lockout_memcached_servers', None, class RequestLogging(wsgi.Middleware): """Access-Log akin logging for all EC2 API requests.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): start = utils.utcnow() rv = req.get_response(self.application) @@ -112,7 +112,7 @@ class Lockout(wsgi.Middleware): debug=0) super(Lockout, self).__init__(application) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): access_key = str(req.params['AWSAccessKeyId']) failures_key = "authfailures-%s" % access_key @@ -141,7 +141,7 @@ class Authenticate(wsgi.Middleware): """Authenticate an EC2 request and add 'ec2.context' to WSGI environ.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): # Read request signature and access id. try: @@ -190,7 +190,7 @@ class Requestify(wsgi.Middleware): super(Requestify, self).__init__(app) self.controller = utils.import_class(controller)() - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): non_args = ['Action', 'Signature', 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Version', 'Timestamp'] @@ -269,7 +269,7 @@ class Authorizer(wsgi.Middleware): }, } - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): context = req.environ['ec2.context'] controller = req.environ['ec2.request'].controller.__class__.__name__ @@ -303,7 +303,7 @@ class Executor(wsgi.Application): response, or a 400 upon failure. """ - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): context = req.environ['ec2.context'] api_request = req.environ['ec2.request'] @@ -365,7 +365,7 @@ class Executor(wsgi.Application): class Versions(wsgi.Application): - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Respond to a request for all EC2 versions.""" # available api versions diff --git a/nova/api/ec2/metadatarequesthandler.py b/nova/api/ec2/metadatarequesthandler.py index 6fb441656..28f99b0ef 100644 --- a/nova/api/ec2/metadatarequesthandler.py +++ b/nova/api/ec2/metadatarequesthandler.py @@ -65,7 +65,7 @@ class MetadataRequestHandler(wsgi.Application): data = data[item] return data - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): cc = cloud.CloudController() remote_address = req.remote_addr diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 274330e3b..b5439788d 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -47,7 +47,7 @@ flags.DEFINE_bool('allow_admin_api', class FaultWrapper(wsgi.Middleware): """Calls down the middleware stack, making exceptions into faults.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): try: return req.get_response(self.application) @@ -115,7 +115,7 @@ class APIRouter(wsgi.Router): class Versions(wsgi.Application): - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Respond to a request for all OpenStack API versions.""" response = { diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 6011e6115..de8905f46 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -46,7 +46,7 @@ class AuthMiddleware(wsgi.Middleware): self.auth = auth.manager.AuthManager() super(AuthMiddleware, self).__init__(application) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): if not self.has_authentication(req): return self.authenticate(req) @@ -121,7 +121,7 @@ class AuthMiddleware(wsgi.Middleware): username - string key - string API key - req - webob.Request object + req - wsgi.Request object """ ctxt = context.get_admin_context() user = self.auth.get_user_from_access_key(key) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 9f85c5c8a..49b970d75 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -25,7 +25,7 @@ def limited(items, request, max_limit=1000): Return a slice of items according to requested offset and limit. @param items: A sliceable entity - @param request: `webob.Request` possibly containing 'offset' and 'limit' + @param request: `wsgi.Request` possibly containing 'offset' and 'limit' GET variables. 'offset' is where to start in the list, and 'limit' is the maximum number of items to return. If 'limit' is not specified, 0, or > max_limit, we default diff --git a/nova/api/openstack/faults.py b/nova/api/openstack/faults.py index 224a7ef0b..c70b00fa3 100644 --- a/nova/api/openstack/faults.py +++ b/nova/api/openstack/faults.py @@ -42,7 +42,7 @@ class Fault(webob.exc.HTTPException): """Create a Fault for the given webob.exc.exception.""" self.wrapped_exc = exception - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Generate a WSGI response based on the exception passed to ctor.""" # Replace the body with fault details. diff --git a/nova/api/openstack/ratelimiting/__init__.py b/nova/api/openstack/ratelimiting/__init__.py index cbb4b897e..88ffc3246 100644 --- a/nova/api/openstack/ratelimiting/__init__.py +++ b/nova/api/openstack/ratelimiting/__init__.py @@ -57,7 +57,7 @@ class RateLimitingMiddleware(wsgi.Middleware): self.limiter = WSGIAppProxy(service_host) super(RateLimitingMiddleware, self).__init__(application) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Rate limit the request. @@ -183,7 +183,7 @@ class WSGIApp(object): """Create the WSGI application using the given Limiter instance.""" self.limiter = limiter - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): parts = req.path_info.split('/') # format: /limiter// diff --git a/nova/wsgi.py b/nova/wsgi.py index 1eb66d067..67216d540 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -82,6 +82,27 @@ class Server(object): log=WritableLogger(logger)) +class Request(webob.Request): + + def best_match(self): + """ + Determine the most acceptable content-type based on the + query extension then the Accept header + """ + + parts = self.path.rsplit(".", 1) + + if len(parts) > 1: + format = parts[1] + if format in ["json", "xml"]: + return "application/{0}".format(parts[1]) + + ctypes = ["application/json", "application/xml"] + bm = self.accept.best_match(ctypes) + + return bm or "application/json" + + class Application(object): """Base WSGI application wrapper. Subclasses need to implement __call__.""" @@ -113,7 +134,7 @@ class Application(object): def __call__(self, environ, start_response): r"""Subclasses will probably want to implement __call__ like this: - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): # Any of the following objects work as responses: @@ -199,7 +220,7 @@ class Middleware(Application): """Do whatever you'd like to the response.""" return response - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): response = self.process_request(req) if response: @@ -212,7 +233,7 @@ class Debug(Middleware): """Helper class that can be inserted into any WSGI application chain to get information about the request and response.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): print ("*" * 40) + " REQUEST ENVIRON" for key, value in req.environ.items(): @@ -276,7 +297,7 @@ class Router(object): self._router = routes.middleware.RoutesMiddleware(self._dispatch, self.map) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): """ Route the incoming request to a controller based on self.map. @@ -285,7 +306,7 @@ class Router(object): return self._router @staticmethod - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def _dispatch(req): """ Called by self._router after matching the incoming request to a route @@ -304,11 +325,11 @@ class Controller(object): WSGI app that reads routing information supplied by RoutesMiddleware and calls the requested action method upon itself. All action methods must, in addition to their normal parameters, accept a 'req' argument - which is the incoming webob.Request. They raise a webob.exc exception, + which is the incoming wsgi.Request. They raise a webob.exc exception, or return a dict which will be serialized by requested content type. """ - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): """ Call the method specified in req.environ by RoutesMiddleware. @@ -358,7 +379,7 @@ class Serializer(object): needed to serialize a dictionary to that type. """ self.metadata = metadata or {} - req = webob.Request.blank('', environ) + req = wsgi.Request.blank('', environ) suffix = req.path_info.split('.')[-1].lower() if suffix == 'json': self.handler = self._to_json -- cgit From 137a4946785b9460aadb9fe40f2b0e18bd7f6063 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Fri, 4 Mar 2011 01:09:21 +0900 Subject: Merged to trunk rev 757. Main changes are below. 1. Rename db table ComputeService -> ComputeNode 2. nova-manage option instance_type is reserved and we cannot use option instance, so change instance -> vm. --- nova/db/api.py | 12 +++++----- nova/db/sqlalchemy/api.py | 26 +++++++++++----------- .../versions/009_add_live_migration.py | 8 +++---- nova/db/sqlalchemy/models.py | 10 ++++----- nova/scheduler/driver.py | 10 ++++----- nova/scheduler/manager.py | 14 ++++++------ nova/tests/test_scheduler.py | 16 ++++++------- nova/tests/test_virt.py | 22 +++++++++--------- nova/virt/libvirt_conn.py | 10 ++++----- 9 files changed, 64 insertions(+), 64 deletions(-) (limited to 'nova') diff --git a/nova/db/api.py b/nova/db/api.py index 13bc07ad2..3b427cefe 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -153,24 +153,24 @@ def service_update(context, service_id, values): ################### -def compute_service_get(context, compute_id, session=None): +def compute_node_get(context, compute_id, session=None): """Get an computeService or raise if it does not exist.""" - return IMPL.compute_service_get(context, compute_id) + return IMPL.compute_node_get(context, compute_id) -def compute_service_create(context, values): +def compute_node_create(context, values): """Create a computeService from the values dictionary.""" - return IMPL.compute_service_create(context, values) + return IMPL.compute_node_create(context, values) -def compute_service_update(context, compute_id, values): +def compute_node_update(context, compute_id, values): """Set the given properties on an computeService and update it. Raises NotFound if computeService does not exist. """ - return IMPL.compute_service_update(context, compute_id, values) + return IMPL.compute_node_update(context, compute_id, values) ################### diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index bed621b18..69aa07279 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -119,8 +119,8 @@ def service_destroy(context, service_id): service_ref.delete(session=session) if service_ref.topic == 'compute' and \ - len(service_ref.compute_service) != 0: - for c in service_ref.compute_service: + len(service_ref.compute_node) != 0: + for c in service_ref.compute_node: c.delete(session=session) @@ -130,7 +130,7 @@ def service_get(context, service_id, session=None): session = get_session() result = session.query(models.Service).\ - options(joinedload('compute_service')).\ + options(joinedload('compute_node')).\ filter_by(id=service_id).\ filter_by(deleted=can_read_deleted(context)).\ first() @@ -174,7 +174,7 @@ def service_get_all_compute_by_host(context, host): topic = 'compute' session = get_session() result = session.query(models.Service).\ - options(joinedload('compute_service')).\ + options(joinedload('compute_node')).\ filter_by(deleted=False).\ filter_by(host=host).\ filter_by(topic=topic).\ @@ -298,11 +298,11 @@ def service_update(context, service_id, values): @require_admin_context -def compute_service_get(context, compute_id, session=None): +def compute_node_get(context, compute_id, session=None): if not session: session = get_session() - result = session.query(models.ComputeService).\ + result = session.query(models.ComputeNode).\ filter_by(id=compute_id).\ filter_by(deleted=can_read_deleted(context)).\ first() @@ -314,18 +314,18 @@ def compute_service_get(context, compute_id, session=None): @require_admin_context -def compute_service_create(context, values): - compute_service_ref = models.ComputeService() - compute_service_ref.update(values) - compute_service_ref.save() - return compute_service_ref +def compute_node_create(context, values): + compute_node_ref = models.ComputeNode() + compute_node_ref.update(values) + compute_node_ref.save() + return compute_node_ref @require_admin_context -def compute_service_update(context, compute_id, values): +def compute_node_update(context, compute_id, values): session = get_session() with session.begin(): - compute_ref = compute_service_get(context, compute_id, session=session) + compute_ref = compute_node_get(context, compute_id, session=session) compute_ref.update(values) compute_ref.save(session=session) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/009_add_live_migration.py b/nova/db/sqlalchemy/migrate_repo/versions/009_add_live_migration.py index 2689b5b74..23ccccb4e 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/009_add_live_migration.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/009_add_live_migration.py @@ -31,7 +31,7 @@ instances = Table('instances', meta, # New Tables # -compute_services = Table('compute_services', meta, +compute_nodes = Table('compute_nodes', meta, Column('created_at', DateTime(timezone=False)), Column('updated_at', DateTime(timezone=False)), Column('deleted_at', DateTime(timezone=False)), @@ -73,11 +73,11 @@ def upgrade(migrate_engine): meta.bind = migrate_engine try: - compute_services.create() + compute_nodes.create() except Exception: - logging.info(repr(compute_services)) + logging.info(repr(compute_nodes)) logging.exception('Exception while creating table') - meta.drop_all(tables=[compute_services]) + meta.drop_all(tables=[compute_nodes]) raise instances.create_column(instances_launched_on) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 2af7377ef..8646190f3 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -113,18 +113,18 @@ class Service(BASE, NovaBase): availability_zone = Column(String(255), default='nova') -class ComputeService(BASE, NovaBase): +class ComputeNode(BASE, NovaBase): """Represents a running compute service on a host.""" - __tablename__ = 'compute_services' + __tablename__ = 'compute_nodes' id = Column(Integer, primary_key=True) service_id = Column(Integer, ForeignKey('services.id'), nullable=True) service = relationship(Service, - backref=backref('compute_service'), + backref=backref('compute_node'), foreign_keys=service_id, primaryjoin='and_(' - 'ComputeService.service_id == Service.id,' - 'ComputeService.deleted == False)') + 'ComputeNode.service_id == Service.id,' + 'ComputeNode.deleted == False)') vcpus = Column(Integer, nullable=True) memory_mb = Column(Integer, nullable=True) diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 4485ba39f..791f9000d 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -190,7 +190,7 @@ class Scheduler(object): # Checking dest exists. dservice_refs = db.service_get_all_compute_by_host(context, dest) - dservice_ref = dservice_refs[0]['compute_service'][0] + dservice_ref = dservice_refs[0]['compute_node'][0] # Checking original host( where instance was launched at) exists. try: @@ -200,7 +200,7 @@ class Scheduler(object): raise exception.Invalid(_("host %s where instance was launched " "does not exist.") % instance_ref['launched_on']) - oservice_ref = oservice_refs[0]['compute_service'][0] + oservice_ref = oservice_refs[0]['compute_node'][0] # Checking hypervisor is same. orig_hypervisor = oservice_ref['hypervisor_type'] @@ -252,10 +252,10 @@ class Scheduler(object): # Getting host information service_refs = db.service_get_all_compute_by_host(context, dest) - compute_service_ref = service_refs[0]['compute_service'][0] + compute_node_ref = service_refs[0]['compute_node'][0] - mem_total = int(compute_service_ref['memory_mb']) - mem_used = int(compute_service_ref['memory_mb_used']) + mem_total = int(compute_node_ref['memory_mb']) + mem_used = int(compute_node_ref['memory_mb_used']) mem_avail = mem_total - mem_used mem_inst = instance_ref['memory_mb'] if mem_avail <= mem_inst: diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index a50d3ab20..090d8b89d 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -87,13 +87,13 @@ class SchedulerManager(manager.Manager): compute_ref = compute_ref[0] # Getting physical resource information - compute_service_ref = compute_ref['compute_service'][0] - resource = {'vcpus': compute_service_ref['vcpus'], - 'memory_mb': compute_service_ref['memory_mb'], - 'local_gb': compute_service_ref['local_gb'], - 'vcpus_used': compute_service_ref['vcpus_used'], - 'memory_mb_used': compute_service_ref['memory_mb_used'], - 'local_gb_used': compute_service_ref['local_gb_used']} + compute_node_ref = compute_ref['compute_node'][0] + resource = {'vcpus': compute_node_ref['vcpus'], + 'memory_mb': compute_node_ref['memory_mb'], + 'local_gb': compute_node_ref['local_gb'], + 'vcpus_used': compute_node_ref['vcpus_used'], + 'memory_mb_used': compute_node_ref['memory_mb_used'], + 'local_gb_used': compute_node_ref['local_gb_used']} # Getting usage resource information usage = {} diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 62db42b11..711b66af7 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -60,7 +60,7 @@ class SchedulerTestCase(test.TestCase): self.flags(scheduler_driver='nova.tests.test_scheduler.TestDriver') def _create_compute_service(self): - """Create compute-manager(ComputeService and Service record).""" + """Create compute-manager(ComputeNode and Service record).""" ctxt = context.get_admin_context() dic = {'host': 'dummy', 'binary': 'nova-compute', 'topic': 'compute', 'report_count': 0, 'availability_zone': 'dummyzone'} @@ -71,7 +71,7 @@ class SchedulerTestCase(test.TestCase): 'vcpus_used': 16, 'memory_mb_used': 32, 'local_gb_used': 10, 'hypervisor_type': 'qemu', 'hypervisor_version': 12003, 'cpu_info': ''} - db.compute_service_create(ctxt, dic) + db.compute_node_create(ctxt, dic) return db.service_get(ctxt, s_ref['id']) @@ -144,8 +144,8 @@ class SchedulerTestCase(test.TestCase): # result checking c1 = ('resource' in result and 'usage' in result) - compute_service = s_ref['compute_service'][0] - c2 = self._dic_is_equal(result['resource'], compute_service) + compute_node = s_ref['compute_node'][0] + c2 = self._dic_is_equal(result['resource'], compute_node) c3 = result['usage'] == {} self.assertTrue(c1 and c2 and c3) db.service_destroy(ctxt, s_ref['id']) @@ -163,8 +163,8 @@ class SchedulerTestCase(test.TestCase): result = scheduler.show_host_resources(ctxt, s_ref['host']) c1 = ('resource' in result and 'usage' in result) - compute_service = s_ref['compute_service'][0] - c2 = self._dic_is_equal(result['resource'], compute_service) + compute_node = s_ref['compute_node'][0] + c2 = self._dic_is_equal(result['resource'], compute_node) c3 = result['usage'].keys() == ['p-01', 'p-02'] keys = ['vcpus', 'memory_mb', 'local_gb'] c4 = self._dic_is_equal(result['usage']['p-01'], i_ref1, keys) @@ -301,7 +301,7 @@ class SimpleDriverTestCase(test.TestCase): dic['memory_mb_used'] = kwargs.get('memory_mb_used', 32) dic['hypervisor_type'] = kwargs.get('hypervisor_type', 'qemu') dic['hypervisor_version'] = kwargs.get('hypervisor_version', 12003) - db.compute_service_create(self.context, dic) + db.compute_node_create(self.context, dic) return db.service_get(self.context, s_ref['id']) def test_doesnt_report_disabled_hosts_as_up(self): @@ -923,7 +923,7 @@ class SimpleDriverTestCase(test.TestCase): self.mox.StubOutWithMock(rpc, 'call', use_mock_anything=True) rpc.call(mox.IgnoreArg(), mox.IgnoreArg(), {"method": 'compare_cpu', - "args": {'cpu_info': s_ref2['compute_service'][0]['cpu_info']}}).\ + "args": {'cpu_info': s_ref2['compute_node'][0]['cpu_info']}}).\ AndRaise(rpc.RemoteError("doesn't have compatibility to", "", "")) self.mox.ReplayAll() diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 17b80c294..aac55a894 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -275,7 +275,7 @@ class LibvirtConnTestCase(test.TestCase): db.instance_destroy(user_context, instance_ref['id']) def test_update_available_resource_works_correctly(self): - """Confirm compute_service table is updated successfully.""" + """Confirm compute_node table is updated successfully.""" org_path = FLAGS.instances_path = '' FLAGS.instances_path = '.' @@ -289,16 +289,16 @@ class LibvirtConnTestCase(test.TestCase): conn = libvirt_conn.LibvirtConnection(False) conn.update_available_resource(self.context, 'dummy') service_ref = db.service_get(self.context, service_ref['id']) - compute_service = service_ref['compute_service'][0] - - c1 = (compute_service['vcpus'] > 0) - c2 = (compute_service['memory_mb'] > 0) - c3 = (compute_service['local_gb'] > 0) - c4 = (compute_service['vcpus_used'] == 0) - c5 = (compute_service['memory_mb_used'] > 0) - c6 = (compute_service['local_gb_used'] > 0) - c7 = (len(compute_service['hypervisor_type']) > 0) - c8 = (compute_service['hypervisor_version'] > 0) + compute_node = service_ref['compute_node'][0] + + c1 = (compute_node['vcpus'] > 0) + c2 = (compute_node['memory_mb'] > 0) + c3 = (compute_node['local_gb'] > 0) + c4 = (compute_node['vcpus_used'] == 0) + c5 = (compute_node['memory_mb_used'] > 0) + c6 = (compute_node['local_gb_used'] > 0) + c7 = (len(compute_node['hypervisor_type']) > 0) + c8 = (compute_node['hypervisor_version'] > 0) self.assertTrue(c1 and c2 and c3 and c4 and c5 and c6 and c7 and c8) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index b9abf1890..71ca508b0 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1021,7 +1021,7 @@ class LibvirtConnection(object): self.firewall_driver.refresh_security_group_members(security_group_id) def update_available_resource(self, ctxt, host): - """Updates compute manager resource info on ComputeService table. + """Updates compute manager resource info on ComputeNode table. This method is called when nova-coompute launches, and whenever admin executes "nova-manage service update_resource". @@ -1049,14 +1049,14 @@ class LibvirtConnection(object): 'hypervisor_version': self.get_hypervisor_version(), 'cpu_info': self.get_cpu_info()} - compute_service_ref = service_ref['compute_service'] - if not compute_service_ref: + compute_node_ref = service_ref['compute_node'] + if not compute_node_ref: LOG.info(_('Compute_service record is created for %s ') % host) dic['service_id'] = service_ref['id'] - db.compute_service_create(ctxt, dic) + db.compute_node_create(ctxt, dic) else: LOG.info(_('Compute_service record is updated for %s ') % host) - db.compute_service_update(ctxt, compute_service_ref[0]['id'], dic) + db.compute_node_update(ctxt, compute_node_ref[0]['id'], dic) def compare_cpu(self, cpu_info): """Checks the host cpu is compatible to a cpu given by xml. -- cgit From bc6cc457132b096150dcd9ff2ed2909585a80484 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Fri, 4 Mar 2011 01:17:05 +0900 Subject: some comments are modified --- nova/db/api.py | 8 ++++---- nova/db/sqlalchemy/api.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'nova') diff --git a/nova/db/api.py b/nova/db/api.py index 3b427cefe..43e1c2183 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -154,19 +154,19 @@ def service_update(context, service_id, values): def compute_node_get(context, compute_id, session=None): - """Get an computeService or raise if it does not exist.""" + """Get an computeNode or raise if it does not exist.""" return IMPL.compute_node_get(context, compute_id) def compute_node_create(context, values): - """Create a computeService from the values dictionary.""" + """Create a computeNode from the values dictionary.""" return IMPL.compute_node_create(context, values) def compute_node_update(context, compute_id, values): - """Set the given properties on an computeService and update it. + """Set the given properties on an computeNode and update it. - Raises NotFound if computeService does not exist. + Raises NotFound if computeNode does not exist. """ diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 69aa07279..b305543ff 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -308,7 +308,7 @@ def compute_node_get(context, compute_id, session=None): first() if not result: - raise exception.NotFound(_('No computeService for id %s') % compute_id) + raise exception.NotFound(_('No computeNode for id %s') % compute_id) return result -- cgit From 6d075754bdd4090342bf4f79c726a52923c311a8 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 3 Mar 2011 12:45:34 -0500 Subject: adding wsgi.Controller and wsgi.Request testing; fixing format keyword argument exception --- nova/tests/api/test_wsgi.py | 120 +++++++++++++++++++++++++++++++++++++------- nova/wsgi.py | 4 +- 2 files changed, 105 insertions(+), 19 deletions(-) (limited to 'nova') diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index 2c7852214..cf2d0e297 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -21,11 +21,13 @@ Test WSGI basics and provide some helper functions for other WSGI tests. """ +import json from nova import test import routes import webob +from nova import exception from nova import wsgi @@ -66,30 +68,112 @@ class Test(test.TestCase): result = webob.Request.blank('/bad').get_response(Router()) self.assertNotEqual(result.body, "Router result") - def test_controller(self): - class Controller(wsgi.Controller): - """Test controller to call from router.""" - test = self +class ControllerTest(test.TestCase): + + class TestRouter(wsgi.Router): + + class TestController(wsgi.Controller): + + _serialization_metadata = { + 'application/xml': { + "attributes": { + "test": ["id"]}}} def show(self, req, id): # pylint: disable-msg=W0622,C0103 - """Default action called for requests with an ID.""" - self.test.assertEqual(req.path_info, '/tests/123') - self.test.assertEqual(id, '123') - return id + return {"test": {"id": id}} + + def __init__(self): + mapper = routes.Mapper() + mapper.resource("test", "tests", controller=self.TestController()) + wsgi.Router.__init__(self, mapper) + + def test_show(self): + request = wsgi.Request.blank('/tests/123') + result = request.get_response(self.TestRouter()) + self.assertEqual(json.loads(result.body), {"test": {"id": "123"}}) + + def test_content_type_from_accept_xml(self): + request = webob.Request.blank('/tests/123') + request.headers["Accept"] = "application/xml" + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/xml") + + def test_content_type_from_accept_json(self): + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/json" + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/json") + + def test_content_type_from_query_extension_xml(self): + request = wsgi.Request.blank('/tests/123.xml') + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/xml") + + def test_content_type_from_query_extension_json(self): + request = wsgi.Request.blank('/tests/123.json') + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/json") + + def test_content_type_default_when_unsupported(self): + request = wsgi.Request.blank('/tests/123.unsupported') + request.headers["Accept"] = "application/unsupported1" + result = request.get_response(self.TestRouter()) + self.assertEqual(result.status_int, 200) + self.assertEqual(result.headers["Content-Type"], "application/json") + + +class RequestTest(test.TestCase): + + def test_content_type_from_accept_xml(self): + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/xml" + result = request.best_match() + self.assertEqual(result, "application/xml") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/json" + result = request.best_match() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/xml, application/json" + result = request.best_match() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = \ + "application/json; q=0.3, application/xml; q=0.9" + result = request.best_match() + self.assertEqual(result, "application/xml") + + def test_content_type_from_query_extension(self): + request = wsgi.Request.blank('/tests/123.xml') + result = request.best_match() + self.assertEqual(result, "application/xml") + + request = wsgi.Request.blank('/tests/123.json') + result = request.best_match() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123.invalid') + result = request.best_match() + self.assertEqual(result, "application/json") + + def test_content_type_accept_and_query_extension(self): + request = wsgi.Request.blank('/tests/123.xml') + request.headers["Accept"] = "application/json" + result = request.best_match() + self.assertEqual(result, "application/xml") + + def test_content_type_accept_default(self): + request = wsgi.Request.blank('/tests/123.unsupported') + request.headers["Accept"] = "application/unsupported1" + result = request.best_match() + self.assertEqual(result, "application/json") - class Router(wsgi.Router): - """Test router.""" - def __init__(self): - mapper = routes.Mapper() - mapper.resource("test", "tests", controller=Controller()) - super(Router, self).__init__(mapper) - result = webob.Request.blank('/tests/123').get_response(Router()) - self.assertEqual(result.body, "123") - result = webob.Request.blank('/test/123').get_response(Router()) - self.assertNotEqual(result.body, "123") class SerializerTest(test.TestCase): diff --git a/nova/wsgi.py b/nova/wsgi.py index 67216d540..4577439cb 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -339,6 +339,8 @@ class Controller(object): method = getattr(self, action) del arg_dict['controller'] del arg_dict['action'] + if 'format' in arg_dict: + del arg_dict['format'] arg_dict['req'] = req result = method(**arg_dict) if type(result) is dict: @@ -379,7 +381,7 @@ class Serializer(object): needed to serialize a dictionary to that type. """ self.metadata = metadata or {} - req = wsgi.Request.blank('', environ) + req = Request.blank('', environ) suffix = req.path_info.split('.')[-1].lower() if suffix == 'json': self.handler = self._to_json -- cgit From 9cfe8ff2e8e66952c3202b852a88ee6fca6fb736 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 3 Mar 2011 16:31:01 -0500 Subject: pep8 --- nova/api/openstack/servers.py | 9 +++++---- nova/tests/api/openstack/test_servers.py | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 73c787828..ea13116fa 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -85,6 +85,7 @@ def _translate_detail_keys(inst): return dict(server=inst_dict) + def _translate_keys(inst): """ Coerces into dictionary format, excluding all model attributes save for id and name """ @@ -143,8 +144,8 @@ class Controller(wsgi.Controller): """ Create a list of onset files from the personality request attribute - At this time, onset_files must be formatted as a list of - (file_path, file_content) pairs for compatibility with the + At this time, onset_files must be formatted as a list of + (file_path, file_content) pairs for compatibility with the underlying compute service. """ onset_files = [] @@ -157,8 +158,8 @@ class Controller(wsgi.Controller): try: contents = base64.b64decode(contents) except TypeError: - raise exc.HTTPBadRequest(explanation= - 'Personality content for %s cannot be decoded' % path) + msg = 'Personality content for %s cannot be decoded' % path + raise exc.HTTPBadRequest(explanation=msg) onset_files.append((path, contents)) return onset_files diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 272d34e3a..53cfa3a6e 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -287,8 +287,8 @@ class ServersTest(test.TestCase): request, response, onset_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [(path, contents)]) - + self.assertEquals(onset_files, [(path, contents)]) + def test_create_instance_with_personality_with_non_b64_content(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Oh no!"\n' @@ -320,7 +320,7 @@ class ServersTest(test.TestCase): request, response, onset_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [(path, contents)]) + self.assertEquals(onset_files, [(path, contents)]) def test_create_instance_personality_not_a_list(self): path = '/my/file/path' -- cgit From 848aced747a60c47d76efcb2147041339df4a628 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 3 Mar 2011 17:21:21 -0500 Subject: Refactor wsgi.Serializer away from handling Requests directly; now require Content-Type in all requests; fix tests according to new code --- nova/api/direct.py | 2 +- nova/api/openstack/__init__.py | 4 +- nova/api/openstack/consoles.py | 2 +- nova/api/openstack/faults.py | 5 +- nova/api/openstack/images.py | 2 +- nova/api/openstack/servers.py | 9 ++- nova/api/openstack/zones.py | 4 +- nova/exception.py | 4 ++ nova/tests/api/openstack/test_servers.py | 1 + nova/tests/api/openstack/test_zones.py | 15 +++-- nova/tests/api/test_wsgi.py | 95 +++++++++++++++++++------------ nova/tests/test_direct.py | 3 + nova/wsgi.py | 96 ++++++++++++++++++++------------ 13 files changed, 152 insertions(+), 90 deletions(-) (limited to 'nova') diff --git a/nova/api/direct.py b/nova/api/direct.py index cd237f649..1d699f947 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -206,7 +206,7 @@ class ServiceWrapper(wsgi.Controller): params = dict([(str(k), v) for (k, v) in params.iteritems()]) result = method(context, **params) if type(result) is dict or type(result) is list: - return self._serialize(result, req) + return self._serialize(result, req.best_match()) else: return result diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index b5439788d..6e1a2a06c 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -124,4 +124,6 @@ class Versions(wsgi.Application): metadata = { "application/xml": { "attributes": dict(version=["status", "id"])}} - return wsgi.Serializer(req.environ, metadata).to_content_type(response) + + content_type = req.best_match() + return wsgi.Serializer(metadata).serialize(response, content_type) diff --git a/nova/api/openstack/consoles.py b/nova/api/openstack/consoles.py index 9ebdbe710..8c291c2eb 100644 --- a/nova/api/openstack/consoles.py +++ b/nova/api/openstack/consoles.py @@ -65,7 +65,7 @@ class Controller(wsgi.Controller): def create(self, req, server_id): """Creates a new console""" - #info = self._deserialize(req.body, req) + #info = self._deserialize(req.body, req.get_content_type()) self.console_api.create_console( req.environ['nova.context'], int(server_id)) diff --git a/nova/api/openstack/faults.py b/nova/api/openstack/faults.py index c70b00fa3..075fdb997 100644 --- a/nova/api/openstack/faults.py +++ b/nova/api/openstack/faults.py @@ -57,6 +57,7 @@ class Fault(webob.exc.HTTPException): fault_data[fault_name]['retryAfter'] = retry # 'code' is an attribute on the fault tag itself metadata = {'application/xml': {'attributes': {fault_name: 'code'}}} - serializer = wsgi.Serializer(req.environ, metadata) - self.wrapped_exc.body = serializer.to_content_type(fault_data) + serializer = wsgi.Serializer(metadata) + content_type = req.best_match() + self.wrapped_exc.body = serializer.serialize(fault_data, content_type) return self.wrapped_exc diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index cf85a496f..98f0dd96b 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -151,7 +151,7 @@ class Controller(wsgi.Controller): def create(self, req): context = req.environ['nova.context'] - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) instance_id = env["image"]["serverId"] name = env["image"]["name"] diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 08b95b46a..24d2826af 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -141,7 +141,7 @@ class Controller(wsgi.Controller): def create(self, req): """ Creates a new server for a given user """ - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) if not env: return faults.Fault(exc.HTTPUnprocessableEntity()) @@ -182,7 +182,10 @@ class Controller(wsgi.Controller): def update(self, req, id): """ Updates the server name or password """ - inst_dict = self._deserialize(req.body, req) + if len(req.body) == 0: + raise exc.HTTPUnprocessableEntity() + + inst_dict = self._deserialize(req.body, req.get_content_type()) if not inst_dict: return faults.Fault(exc.HTTPUnprocessableEntity()) @@ -205,7 +208,7 @@ class Controller(wsgi.Controller): def action(self, req, id): """ Multi-purpose method used to reboot, rebuild, and resize a server """ - input_dict = self._deserialize(req.body, req) + input_dict = self._deserialize(req.body, req.get_content_type()) #TODO(sandy): rebuild/resize not supported. try: reboot_type = input_dict['reboot']['type'] diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index d5206da20..cf6cd789f 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -67,13 +67,13 @@ class Controller(wsgi.Controller): def create(self, req): context = req.environ['nova.context'] - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) zone = db.zone_create(context, env["zone"]) return dict(zone=_scrub_zone(zone)) def update(self, req, id): context = req.environ['nova.context'] - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) zone_id = int(id) zone = db.zone_update(context, zone_id, env["zone"]) return dict(zone=_scrub_zone(zone)) diff --git a/nova/exception.py b/nova/exception.py index 7d65bd6a5..93c5fe3d7 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -88,6 +88,10 @@ class InvalidInputException(Error): pass +class InvalidContentType(Error): + pass + + class TimeoutException(Error): pass diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 78beb7df9..fae08d0be 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -227,6 +227,7 @@ class ServersTest(test.TestCase): req = webob.Request.blank('/v1.0/servers') req.method = 'POST' req.body = json.dumps(body) + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 555b206b9..d0da8eaaf 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -86,24 +86,27 @@ class ZonesTest(test.TestCase): def test_get_zone_list(self): req = webob.Request.blank('/v1.0/zones') + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(len(res_dict['zones']), 2) def test_get_zone_by_id(self): req = webob.Request.blank('/v1.0/zones/1') + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) + self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://foo.com') self.assertFalse('password' in res_dict['zone']) - self.assertEqual(res.status_int, 200) def test_zone_delete(self): req = webob.Request.blank('/v1.0/zones/1') + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) @@ -112,13 +115,14 @@ class ZonesTest(test.TestCase): body = dict(zone=dict(api_url='http://blah.zoo', username='fred', password='fubar')) req = webob.Request.blank('/v1.0/zones') + req.headers["Content-Type"] = "application/json" req.method = 'POST' req.body = json.dumps(body) res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://blah.zoo') self.assertFalse('username' in res_dict['zone']) @@ -126,13 +130,14 @@ class ZonesTest(test.TestCase): def test_zone_update(self): body = dict(zone=dict(username='zeb', password='sneaky')) req = webob.Request.blank('/v1.0/zones/1') + req.headers["Content-Type"] = "application/json" req.method = 'PUT' req.body = json.dumps(body) res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://foo.com') self.assertFalse('username' in res_dict['zone']) diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index cf2d0e297..7c0135656 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -93,29 +93,29 @@ class ControllerTest(test.TestCase): result = request.get_response(self.TestRouter()) self.assertEqual(json.loads(result.body), {"test": {"id": "123"}}) - def test_content_type_from_accept_xml(self): + def test_response_content_type_from_accept_xml(self): request = webob.Request.blank('/tests/123') request.headers["Accept"] = "application/xml" result = request.get_response(self.TestRouter()) self.assertEqual(result.headers["Content-Type"], "application/xml") - def test_content_type_from_accept_json(self): + def test_response_content_type_from_accept_json(self): request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/json" result = request.get_response(self.TestRouter()) self.assertEqual(result.headers["Content-Type"], "application/json") - def test_content_type_from_query_extension_xml(self): + def test_response_content_type_from_query_extension_xml(self): request = wsgi.Request.blank('/tests/123.xml') result = request.get_response(self.TestRouter()) self.assertEqual(result.headers["Content-Type"], "application/xml") - def test_content_type_from_query_extension_json(self): + def test_response_content_type_from_query_extension_json(self): request = wsgi.Request.blank('/tests/123.json') result = request.get_response(self.TestRouter()) self.assertEqual(result.headers["Content-Type"], "application/json") - def test_content_type_default_when_unsupported(self): + def test_response_content_type_default_when_unsupported(self): request = wsgi.Request.blank('/tests/123.unsupported') request.headers["Accept"] = "application/unsupported1" result = request.get_response(self.TestRouter()) @@ -125,6 +125,17 @@ class ControllerTest(test.TestCase): class RequestTest(test.TestCase): + def test_request_content_type_missing(self): + request = wsgi.Request.blank('/tests/123') + request.body = "" + self.assertRaises(webob.exc.HTTPBadRequest, request.get_content_type) + + def test_request_content_type_unsupported(self): + request = wsgi.Request.blank('/tests/123') + request.headers["Content-Type"] = "text/html" + request.body = "asdf
" + self.assertRaises(webob.exc.HTTPBadRequest, request.get_content_type) + def test_content_type_from_accept_xml(self): request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/xml" @@ -173,40 +184,48 @@ class RequestTest(test.TestCase): self.assertEqual(result, "application/json") - - - class SerializerTest(test.TestCase): - def match(self, url, accept, expect): + def test_xml(self): input_dict = dict(servers=dict(a=(2, 3))) expected_xml = '(2,3)' + serializer = wsgi.Serializer() + result = serializer.serialize(input_dict, "application/xml") + result = result.replace('\n', '').replace(' ', '') + self.assertEqual(result, expected_xml) + + def test_json(self): + input_dict = dict(servers=dict(a=(2, 3))) expected_json = '{"servers":{"a":[2,3]}}' - req = webob.Request.blank(url, headers=dict(Accept=accept)) - result = wsgi.Serializer(req.environ).to_content_type(input_dict) + serializer = wsgi.Serializer() + result = serializer.serialize(input_dict, "application/json") result = result.replace('\n', '').replace(' ', '') - if expect == 'xml': - self.assertEqual(result, expected_xml) - elif expect == 'json': - self.assertEqual(result, expected_json) - else: - raise "Bad expect value" - - def test_basic(self): - self.match('/servers/4.json', None, expect='json') - self.match('/servers/4', 'application/json', expect='json') - self.match('/servers/4', 'application/xml', expect='xml') - self.match('/servers/4.xml', None, expect='xml') - - def test_defaults_to_json(self): - self.match('/servers/4', None, expect='json') - self.match('/servers/4', 'text/html', expect='json') - - def test_suffix_takes_precedence_over_accept_header(self): - self.match('/servers/4.xml', 'application/json', expect='xml') - self.match('/servers/4.xml.', 'application/json', expect='json') - - def test_deserialize(self): + self.assertEqual(result, expected_json) + + def test_unsupported_content_type(self): + serializer = wsgi.Serializer() + self.assertRaises(exception.InvalidContentType, serializer.serialize, + {}, "text/null") + + def test_deserialize_json(self): + data = """{"a": { + "a1": "1", + "a2": "2", + "bs": ["1", "2", "3", {"c": {"c1": "1"}}], + "d": {"e": "1"}, + "f": "1"}}""" + as_dict = dict(a={ + 'a1': '1', + 'a2': '2', + 'bs': ['1', '2', '3', {'c': dict(c1='1')}], + 'd': {'e': '1'}, + 'f': '1'}) + metadata = {} + serializer = wsgi.Serializer(metadata) + self.assertEqual(serializer.deserialize(data, "application/json"), + as_dict) + + def test_deserialize_xml(self): xml = """ 123 @@ -221,11 +240,13 @@ class SerializerTest(test.TestCase): 'd': {'e': '1'}, 'f': '1'}) metadata = {'application/xml': dict(plurals={'bs': 'b', 'ts': 't'})} - serializer = wsgi.Serializer({}, metadata) - self.assertEqual(serializer.deserialize(xml), as_dict) + serializer = wsgi.Serializer(metadata) + self.assertEqual(serializer.deserialize(xml, "application/xml"), + as_dict) def test_deserialize_empty_xml(self): xml = """""" as_dict = {"a": {}} - serializer = wsgi.Serializer({}) - self.assertEqual(serializer.deserialize(xml), as_dict) + serializer = wsgi.Serializer() + self.assertEqual(serializer.deserialize(xml, "application/xml"), + as_dict) diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index b6bfab534..85bfcfd85 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -59,6 +59,7 @@ class DirectTestCase(test.TestCase): req.headers['X-OpenStack-User'] = 'user1' req.headers['X-OpenStack-Project'] = 'proj1' resp = req.get_response(self.auth_router) + self.assertEqual(resp.status_int, 200) data = json.loads(resp.body) self.assertEqual(data['user'], 'user1') self.assertEqual(data['project'], 'proj1') @@ -69,6 +70,7 @@ class DirectTestCase(test.TestCase): req.method = 'POST' req.body = 'json=%s' % json.dumps({'data': 'foo'}) resp = req.get_response(self.router) + self.assertEqual(resp.status_int, 200) resp_parsed = json.loads(resp.body) self.assertEqual(resp_parsed['data'], 'foo') @@ -78,6 +80,7 @@ class DirectTestCase(test.TestCase): req.method = 'POST' req.body = 'data=foo' resp = req.get_response(self.router) + self.assertEqual(resp.status_int, 200) resp_parsed = json.loads(resp.body) self.assertEqual(resp_parsed['data'], 'foo') diff --git a/nova/wsgi.py b/nova/wsgi.py index 4577439cb..c3e08522d 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -36,6 +36,7 @@ import webob.exc from paste import deploy +from nova import exception from nova import flags from nova import log as logging from nova import utils @@ -102,6 +103,14 @@ class Request(webob.Request): return bm or "application/json" + def get_content_type(self): + try: + ct = self.headers["Content-Type"] + assert ct in ("application/xml", "application/json") + return ct + except Exception: + raise webob.exc.HTTPBadRequest("Invalid content type") + class Application(object): """Base WSGI application wrapper. Subclasses need to implement __call__.""" @@ -343,30 +352,41 @@ class Controller(object): del arg_dict['format'] arg_dict['req'] = req result = method(**arg_dict) + if type(result) is dict: - return self._serialize(result, req) + content_type = req.best_match() + body = self._serialize(result, content_type) + + response = webob.Response() + response.headers["Content-Type"] = content_type + response.body = body + return response + else: return result - def _serialize(self, data, request): + def _serialize(self, data, content_type): """ - Serialize the given dict to the response type requested in request. + Serialize the given dict to the provided content_type. Uses self._serialization_metadata if it exists, which is a dict mapping MIME types to information needed to serialize to that type. """ _metadata = getattr(type(self), "_serialization_metadata", {}) - serializer = Serializer(request.environ, _metadata) - return serializer.to_content_type(data) + serializer = Serializer(_metadata) + try: + return serializer.serialize(data, content_type) + except exception.InvalidContentType: + raise webob.exc.HTTPNotAcceptable() - def _deserialize(self, data, request): + def _deserialize(self, data, content_type): """ - Deserialize the request body to the response type requested in request. + Deserialize the request body to the specefied content type. Uses self._serialization_metadata if it exists, which is a dict mapping MIME types to information needed to serialize to that type. """ _metadata = getattr(type(self), "_serialization_metadata", {}) - serializer = Serializer(request.environ, _metadata) - return serializer.deserialize(data) + serializer = Serializer(_metadata) + return serializer.deserialize(data, content_type) class Serializer(object): @@ -374,50 +394,52 @@ class Serializer(object): Serializes and deserializes dictionaries to certain MIME types. """ - def __init__(self, environ, metadata=None): + def __init__(self, metadata=None): """ Create a serializer based on the given WSGI environment. 'metadata' is an optional dict mapping MIME types to information needed to serialize a dictionary to that type. """ self.metadata = metadata or {} - req = Request.blank('', environ) - suffix = req.path_info.split('.')[-1].lower() - if suffix == 'json': - self.handler = self._to_json - elif suffix == 'xml': - self.handler = self._to_xml - elif 'application/json' in req.accept: - self.handler = self._to_json - elif 'application/xml' in req.accept: - self.handler = self._to_xml - else: - # This is the default - self.handler = self._to_json - def to_content_type(self, data): - """ - Serialize a dictionary into a string. + def _get_serialize_handler(self, content_type): + handlers = { + "application/json": self._to_json, + "application/xml": self._to_xml, + } + + try: + return handlers[content_type] + except Exception: + raise exception.InvalidContentType() - The format of the string will be decided based on the Content Type - requested in self.environ: by Accept: header, or by URL suffix. + def serialize(self, data, content_type): + """ + Serialize a dictionary into a string of the specified content type. """ - return self.handler(data) + return self._get_serialize_handler(content_type)(data) - def deserialize(self, datastring): + def deserialize(self, datastring, content_type): """ Deserialize a string to a dictionary. The string must be in the format of a supported MIME type. """ - datastring = datastring.strip() + return self.get_deserialize_handler(content_type)(datastring) + + def get_deserialize_handler(self, content_type): + handlers = { + "application/json": self._from_json, + "application/xml": self._from_xml, + } + try: - is_xml = (datastring[0] == '<') - if not is_xml: - return utils.loads(datastring) - return self._from_xml(datastring) - except: - return None + return handlers[content_type] + except Exception: + raise exception.InvalidContentType() + + def _from_json(self, datastring): + return utils.loads(datastring) def _from_xml(self, datastring): xmldata = self.metadata.get('application/xml', {}) -- cgit From 417f6ca5c54878a6bea4d545126f93ecb6a043b4 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Thu, 3 Mar 2011 22:22:00 +0000 Subject: localize a few error messages. --- nova/api/openstack/accounts.py | 2 +- nova/api/openstack/users.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/accounts.py b/nova/api/openstack/accounts.py index 264fdab99..3b90d2776 100644 --- a/nova/api/openstack/accounts.py +++ b/nova/api/openstack/accounts.py @@ -47,7 +47,7 @@ class Controller(wsgi.Controller): """ We cannot depend on the db layer to check for admin access for the auth manager, so we do it here """ if not context.is_admin: - raise exception.NotAuthorized("Not admin user.") + raise exception.NotAuthorized(_("Not admin user.")) def show(self, req, id): """Return data about the given account id""" diff --git a/nova/api/openstack/users.py b/nova/api/openstack/users.py index c0b7544f9..ae3bf7791 100644 --- a/nova/api/openstack/users.py +++ b/nova/api/openstack/users.py @@ -48,7 +48,7 @@ class Controller(wsgi.Controller): """ We cannot depend on the db layer to check for admin access for the auth manager, so we do it here """ if not context.is_admin: - raise exception.NotAuthorized("Not admin user") + raise exception.NotAuthorized(_("Not admin user")) def index(self, req, **kw): """Return all users in brief""" -- cgit From e14f524eb92ae07704a2ec7dac0f97c60940a6ab Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 3 Mar 2011 17:49:41 -0500 Subject: enforce personality quotas --- nova/compute/api.py | 24 ++++++++++++++++++ nova/quota.py | 27 ++++++++++++-------- nova/tests/test_quota.py | 66 +++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 101 insertions(+), 16 deletions(-) (limited to 'nova') diff --git a/nova/compute/api.py b/nova/compute/api.py index 625778b66..44e583cd4 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -80,6 +80,26 @@ class API(base.Base): topic, {"method": "get_network_topic", "args": {'fake': 1}}) + def _check_personality_file_quota(self, context, personality_files): + limit = quota.allowed_personality_files(context) + if len(personality_files) > limit: + raise quota.QuotaError(_("Personality limit exceeded. You can " + "only have %d personalities when " + "creating an instance.") % limit, + "PersonalityLimitExceeded") + path_limit = quota.allowed_personality_path_bytes(context) + content_limit = quota.allowed_personality_content_bytes(context) + for path, content in personality_files: + if len(path) > path_limit: + raise quota.QuotaError( + _("Personality file path limit exceeded."), + "PersonalityLimitExceeded") + if len(content) > content_limit: + raise quota.QuotaError( + _("Personality file content limit exceeded."), + "PersonalityLimitExceeded") + return personality_files + def create(self, context, instance_type, image_id, kernel_id=None, ramdisk_id=None, min_count=1, max_count=1, @@ -124,6 +144,10 @@ class API(base.Base): LOG.warn(msg) raise quota.QuotaError(msg, "MetadataLimitExceeded") + if onset_files is not None: + onset_files = \ + self._check_personality_file_quota(context, onset_files) + image = self.image_service.show(context, image_id) if kernel_id is None: kernel_id = image.get('kernel_id', None) diff --git a/nova/quota.py b/nova/quota.py index 14b388794..4b777624c 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -37,10 +37,12 @@ flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') flags.DEFINE_integer('quota_metadata_items', 128, 'number of metadata items allowed per instance') -flags.DEFINE_integer('quota_file_injection_max_files', 5, - 'number of files allowed during file injection') -flags.DEFINE_integer('quota_file_injection_max_file_bytes', 10 * 1024, - 'number of bytes allowed per file during file injection') +flags.DEFINE_integer('quota_personality_max_files', 5, + 'number of personality files allowed') +flags.DEFINE_integer('quota_personality_max_content_bytes', 10 * 1024, + 'number of bytes allowed per personality file') +flags.DEFINE_integer('quota_personality_max_path_bytes', 255, + 'number of bytes allowed per personality file path') def get_quota(context, project_id): @@ -113,14 +115,19 @@ def allowed_metadata_items(context, num_metadata_items): return min(num_metadata_items, num_allowed_metadata_items) -def allowed_file_injection_files(context): - """Return the number of files allowed per file injection""" - return FLAGS.quota_file_injection_max_files +def allowed_personality_files(context): + """Return the number of personality files allowed""" + return FLAGS.quota_personality_max_files -def allowed_file_injection_file_bytes(context): - """Return the number of bytes allowed per file during injection""" - return FLAGS.quota_file_injection_max_file_bytes +def allowed_personality_content_bytes(context): + """Return the number of bytes allowed per personality content""" + return FLAGS.quota_personality_max_content_bytes + + +def allowed_personality_path_bytes(context): + """Return the number of bytes allowed in a personality file path""" + return FLAGS.quota_personality_max_path_bytes class QuotaError(exception.ApiError): diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 48e5a5538..16a083788 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -177,12 +177,66 @@ class QuotaTestCase(test.TestCase): image_id='fake', metadata=metadata) - def test_allowed_file_injection_files(self): + def test_allowed_personality_files(self): self.assertEqual( - quota.allowed_file_injection_files(self.context), - FLAGS.quota_file_injection_max_files) + quota.allowed_personality_files(self.context), + FLAGS.quota_personality_max_files) + + def _create_with_personality(self, files): + api = compute.API() + api.create(self.context, min_count=1, max_count=1, + instance_type='m1.small', image_id='fake', + onset_files=files) + + def test_no_personality_files(self): + api = compute.API() + api.create(self.context, instance_type='m1.small', image_id='fake') + + def test_max_personality_files(self): + files = [] + for i in xrange(FLAGS.quota_personality_max_files): + files.append(('/my/path%d' % i, 'config = test\n')) + self._create_with_personality(files) # no QuotaError + + def test_too_many_personality_files(self): + files = [] + for i in xrange(FLAGS.quota_personality_max_files + 1): + files.append(('/my/path%d' % i, 'my\ncontent%d\n' % i)) + self.assertRaises(quota.QuotaError, + self._create_with_personality, files) - def test_allowed_file_injection_file_bytes(self): + def test_allowed_personality_content_bytes(self): self.assertEqual( - quota.allowed_file_injection_file_bytes(self.context), - FLAGS.quota_file_injection_max_file_bytes) + quota.allowed_personality_content_bytes(self.context), + FLAGS.quota_personality_max_content_bytes) + + def test_max_personality_content_bytes(self): + max = FLAGS.quota_personality_max_content_bytes + content = ''.join(['a' for i in xrange(max)]) + files = [('/test/path', content)] + self._create_with_personality(files) # no QuotaError + + def test_too_many_personality_content_bytes(self): + max = FLAGS.quota_personality_max_content_bytes + content = ''.join(['a' for i in xrange(max + 1)]) + files = [('/test/path', content)] + self.assertRaises(quota.QuotaError, + self._create_with_personality, files) + + def test_allowed_personality_path_bytes(self): + self.assertEqual( + quota.allowed_personality_path_bytes(self.context), + FLAGS.quota_personality_max_path_bytes) + + def test_max_personality_path_bytes(self): + max = FLAGS.quota_personality_max_path_bytes + path = ''.join(['a' for i in xrange(max)]) + files = [(path, 'config = quotatest')] + self._create_with_personality(files) # no QuotaError + + def test_too_many_personality_path_bytes(self): + max = FLAGS.quota_personality_max_path_bytes + path = ''.join(['a' for i in xrange(max + 1)]) + files = [(path, 'config = quotatest')] + self.assertRaises(quota.QuotaError, + self._create_with_personality, files) -- cgit From c5bfab9a0d213cee549371f05e74747cfcd8f998 Mon Sep 17 00:00:00 2001 From: Ryan Lane Date: Thu, 3 Mar 2011 23:05:00 +0000 Subject: Changing output of status from showing the user as the owner, to showing the project --- nova/api/ec2/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index b1917e9ea..cadda97db 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -562,7 +562,7 @@ class CloudController(object): if context.is_admin: v['status'] = '%s (%s, %s, %s, %s)' % ( volume['status'], - volume['user_id'], + volume['project_id'], volume['host'], instance_data, volume['mountpoint']) -- cgit From 5ae13551990be67e3509ddcd10d1872a91634d83 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 3 Mar 2011 18:27:57 -0500 Subject: rename onset_files to personality_files all the way down to compute manager --- nova/api/openstack/servers.py | 16 +++++++-------- nova/compute/api.py | 10 ++++----- nova/compute/manager.py | 2 +- nova/tests/api/openstack/test_servers.py | 35 ++++++++++++++++---------------- nova/tests/test_quota.py | 2 +- 5 files changed, 33 insertions(+), 32 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index ea13116fa..8f6d8de66 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -140,15 +140,15 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def _get_onset_files_from_personality(self, personality): + def _get_personality_files(self, personality): """ - Create a list of onset files from the personality request attribute + Create a list of personality files from the personality attribute - At this time, onset_files must be formatted as a list of + At this time, personality_files must be formatted as a list of (file_path, file_content) pairs for compatibility with the underlying compute service. """ - onset_files = [] + personality_files = [] for item in personality: try: path = item['path'] @@ -160,8 +160,8 @@ class Controller(wsgi.Controller): except TypeError: msg = 'Personality content for %s cannot be decoded' % path raise exc.HTTPBadRequest(explanation=msg) - onset_files.append((path, contents)) - return onset_files + personality_files.append((path, contents)) + return personality_files def create(self, req): """ Creates a new server for a given user """ @@ -191,7 +191,7 @@ class Controller(wsgi.Controller): metadata.append({'key': k, 'value': v}) personality = env['server'].get('personality', []) - onset_files = self._get_onset_files_from_personality(personality) + personality_files = self._get_personality_files(personality) instances = self.compute_api.create( context, @@ -204,7 +204,7 @@ class Controller(wsgi.Controller): key_name=key_pair['name'], key_data=key_pair['public_key'], metadata=metadata, - onset_files=onset_files) + personality_files=personality_files) return _translate_keys(instances[0]) def update(self, req, id): diff --git a/nova/compute/api.py b/nova/compute/api.py index 44e583cd4..13938dcde 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -106,7 +106,7 @@ class API(base.Base): display_name='', display_description='', key_name=None, key_data=None, security_group='default', availability_zone=None, user_data=None, metadata=[], - onset_files=None): + personality_files=None): """Create the number of instances requested if quota and other arguments check out ok. """ @@ -144,9 +144,9 @@ class API(base.Base): LOG.warn(msg) raise quota.QuotaError(msg, "MetadataLimitExceeded") - if onset_files is not None: - onset_files = \ - self._check_personality_file_quota(context, onset_files) + if personality_files is not None: + personality_files = \ + self._check_personality_file_quota(context, personality_files) image = self.image_service.show(context, image_id) if kernel_id is None: @@ -242,7 +242,7 @@ class API(base.Base): "args": {"topic": FLAGS.compute_topic, "instance_id": instance_id, "availability_zone": availability_zone, - "onset_files": onset_files}}) + "personality_files": personality_files}}) for group_id in security_groups: self.trigger_security_group_members_refresh(elevated, group_id) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index d659712ad..1a392dda8 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -174,7 +174,7 @@ class ComputeManager(manager.Manager): """Launch a new instance with specified options.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) - instance_ref.onset_files = kwargs.get('onset_files', []) + instance_ref.onset_files = kwargs.get('personality_files', []) if instance_ref['name'] in self.driver.list_instances(): raise exception.Error(_("Instance has already been created")) LOG.audit(_("instance %s: starting..."), instance_id, diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 53cfa3a6e..bf934113a 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -241,13 +241,13 @@ class ServersTest(test.TestCase): class FakeComputeAPI(object): def __init__(self): - self.onset_files = None + self.personality_files = None def create(self, *args, **kwargs): - if 'onset_files' in kwargs: - self.onset_files = kwargs['onset_files'] + if 'personality_files' in kwargs: + self.personality_files = kwargs['personality_files'] else: - self.onset_files = None + self.personality_files = None return [{'id': '1234', 'display_name': 'fakeinstance'}] def make_stub_method(canned_return): @@ -271,32 +271,33 @@ class ServersTest(test.TestCase): req = webob.Request.blank('/v1.0/servers') req.method = 'POST' req.body = json.dumps(body) - return req, req.get_response(fakes.wsgi_app()), compute_api.onset_files + return (req, req.get_response(fakes.wsgi_app()), + compute_api.personality_files) def test_create_instance_with_no_personality(self): - request, response, onset_files = \ + request, response, personality_files = \ self._create_instance_with_personality(personality=None) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, []) + self.assertEquals(personality_files, []) def test_create_instance_with_personality(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Hello, World!"\n' b64contents = base64.b64encode(contents) personality = [self._personality_dict(path, b64contents)] - request, response, onset_files = \ + request, response, personality_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [(path, contents)]) + self.assertEquals(personality_files, [(path, contents)]) def test_create_instance_with_personality_with_non_b64_content(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Oh no!"\n' personality = [self._personality_dict(path, contents)] - request, response, onset_files = \ + request, response, personality_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 400) - self.assertEquals(onset_files, None) + self.assertEquals(personality_files, None) def test_create_instance_with_three_personalities(self): files = [ @@ -308,28 +309,28 @@ class ServersTest(test.TestCase): for path, content in files: personality.append(self._personality_dict( path, base64.b64encode(content))) - request, response, onset_files = \ + request, response, personality_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, files) + self.assertEquals(personality_files, files) def test_create_instance_personality_empty_content(self): path = '/my/file/path' contents = '' personality = [self._personality_dict(path, contents)] - request, response, onset_files = \ + request, response, personality_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [(path, contents)]) + self.assertEquals(personality_files, [(path, contents)]) def test_create_instance_personality_not_a_list(self): path = '/my/file/path' contents = 'myfilecontents' personality = self._personality_dict(path, contents) - request, response, onset_files = \ + request, response, personality_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 400) - self.assertEquals(onset_files, None) + self.assertEquals(personality_files, None) def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 16a083788..b26dec61a 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -186,7 +186,7 @@ class QuotaTestCase(test.TestCase): api = compute.API() api.create(self.context, min_count=1, max_count=1, instance_type='m1.small', image_id='fake', - onset_files=files) + personality_files=files) def test_no_personality_files(self): api = compute.API() -- cgit From a433ddeda77aaa4462694661ecdca71eed6db669 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 4 Mar 2011 02:36:55 +0000 Subject: Replace objectstore images with S3 image service backending to glance or local --- nova/api/ec2/cloud.py | 127 +++++++++-------- nova/flags.py | 2 +- nova/image/glance.py | 29 ++-- nova/image/s3.py | 280 ++++++++++++++++++++++++++++---------- nova/image/service.py | 4 +- nova/tests/api/openstack/fakes.py | 11 +- nova/tests/fake_flags.py | 1 + nova/tests/test_cloud.py | 22 ++- nova/tests/test_compute.py | 7 +- nova/tests/test_direct.py | 3 +- nova/tests/test_quota.py | 6 +- 11 files changed, 333 insertions(+), 159 deletions(-) (limited to 'nova') diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 844ccbe5e..8c2e77d86 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -39,7 +39,9 @@ from nova import log as logging from nova import network from nova import utils from nova import volume +from nova.api.ec2 import ec2utils from nova.compute import instance_types +from nova.image import s3 FLAGS = flags.FLAGS @@ -73,30 +75,19 @@ def _gen_key(context, user_id, key_name): return {'private_key': private_key, 'fingerprint': fingerprint} -def ec2_id_to_id(ec2_id): - """Convert an ec2 ID (i-[base 16 number]) to an instance id (int)""" - return int(ec2_id.split('-')[-1], 16) - - -def id_to_ec2_id(instance_id, template='i-%08x'): - """Convert an instance ID (int) to an ec2 ID (i-[base 16 number])""" - return template % instance_id - - class CloudController(object): """ CloudController provides the critical dispatch between inbound API calls through the endpoint and messages sent to the other nodes. """ def __init__(self): - self.image_service = utils.import_object(FLAGS.image_service) + self.image_service = s3.S3ImageService() self.network_api = network.API() self.volume_api = volume.API() self.compute_api = compute.API( network_api=self.network_api, - image_service=self.image_service, volume_api=self.volume_api, - hostname_factory=id_to_ec2_id) + hostname_factory=ec2utils.id_to_ec2_id) self.setup() def __str__(self): @@ -154,7 +145,7 @@ class CloudController(object): availability_zone = self._get_availability_zone_by_host(ctxt, host) floating_ip = db.instance_get_floating_address(ctxt, instance_ref['id']) - ec2_id = id_to_ec2_id(instance_ref['id']) + ec2_id = ec2utils.id_to_ec2_id(instance_ref['id']) data = { 'user-data': base64.b64decode(instance_ref['user_data']), 'meta-data': { @@ -525,7 +516,7 @@ class CloudController(object): ec2_id = instance_id[0] else: ec2_id = instance_id - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) output = self.compute_api.get_console_output( context, instance_id=instance_id) now = datetime.datetime.utcnow() @@ -535,7 +526,7 @@ class CloudController(object): def get_ajax_console(self, context, instance_id, **kwargs): ec2_id = instance_id[0] - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) return self.compute_api.get_ajax_console(context, instance_id=instance_id) @@ -543,7 +534,7 @@ class CloudController(object): if volume_id: volumes = [] for ec2_id in volume_id: - internal_id = ec2_id_to_id(ec2_id) + internal_id = ec2utils.ec2_id_to_id(ec2_id) volume = self.volume_api.get(context, internal_id) volumes.append(volume) else: @@ -556,11 +547,11 @@ class CloudController(object): instance_data = None if volume.get('instance', None): instance_id = volume['instance']['id'] - instance_ec2_id = id_to_ec2_id(instance_id) + instance_ec2_id = ec2utils.id_to_ec2_id(instance_id) instance_data = '%s[%s]' % (instance_ec2_id, volume['instance']['host']) v = {} - v['volumeId'] = id_to_ec2_id(volume['id'], 'vol-%08x') + v['volumeId'] = ec2utils.id_to_ec2_id(volume['id'], 'vol-%08x') v['status'] = volume['status'] v['size'] = volume['size'] v['availabilityZone'] = volume['availability_zone'] @@ -578,8 +569,7 @@ class CloudController(object): 'device': volume['mountpoint'], 'instanceId': instance_ec2_id, 'status': 'attached', - 'volumeId': id_to_ec2_id(volume['id'], - 'vol-%08x')}] + 'volumeId': v['volumeId']}] else: v['attachmentSet'] = [{}] @@ -598,12 +588,12 @@ class CloudController(object): return {'volumeSet': [self._format_volume(context, dict(volume))]} def delete_volume(self, context, volume_id, **kwargs): - volume_id = ec2_id_to_id(volume_id) + volume_id = ec2utils.ec2_id_to_id(volume_id) self.volume_api.delete(context, volume_id=volume_id) return True def update_volume(self, context, volume_id, **kwargs): - volume_id = ec2_id_to_id(volume_id) + volume_id = ec2utils.ec2_id_to_id(volume_id) updatable_fields = ['display_name', 'display_description'] changes = {} for field in updatable_fields: @@ -614,8 +604,8 @@ class CloudController(object): return True def attach_volume(self, context, volume_id, instance_id, device, **kwargs): - volume_id = ec2_id_to_id(volume_id) - instance_id = ec2_id_to_id(instance_id) + volume_id = ec2utils.ec2_id_to_id(volume_id) + instance_id = ec2utils.ec2_id_to_id(instance_id) msg = _("Attach volume %(volume_id)s to instance %(instance_id)s" " at %(device)s") % locals() LOG.audit(msg, context=context) @@ -626,22 +616,22 @@ class CloudController(object): volume = self.volume_api.get(context, volume_id) return {'attachTime': volume['attach_time'], 'device': volume['mountpoint'], - 'instanceId': id_to_ec2_id(instance_id), + 'instanceId': ec2utils.id_to_ec2_id(instance_id), 'requestId': context.request_id, 'status': volume['attach_status'], - 'volumeId': id_to_ec2_id(volume_id, 'vol-%08x')} + 'volumeId': ec2utils.id_to_ec2_id(volume_id, 'vol-%08x')} def detach_volume(self, context, volume_id, **kwargs): - volume_id = ec2_id_to_id(volume_id) + volume_id = ec2utils.ec2_id_to_id(volume_id) LOG.audit(_("Detach volume %s"), volume_id, context=context) volume = self.volume_api.get(context, volume_id) instance = self.compute_api.detach_volume(context, volume_id=volume_id) return {'attachTime': volume['attach_time'], 'device': volume['mountpoint'], - 'instanceId': id_to_ec2_id(instance['id']), + 'instanceId': ec2utils.id_to_ec2_id(instance['id']), 'requestId': context.request_id, 'status': volume['attach_status'], - 'volumeId': id_to_ec2_id(volume_id, 'vol-%08x')} + 'volumeId': ec2utils.id_to_ec2_id(volume_id, 'vol-%08x')} def _convert_to_set(self, lst, label): if lst == None or lst == []: @@ -675,7 +665,7 @@ class CloudController(object): if instance_id: instances = [] for ec2_id in instance_id: - internal_id = ec2_id_to_id(ec2_id) + internal_id = ec2utils.ec2_id_to_id(ec2_id) instance = self.compute_api.get(context, instance_id=internal_id) instances.append(instance) @@ -687,7 +677,7 @@ class CloudController(object): continue i = {} instance_id = instance['id'] - ec2_id = id_to_ec2_id(instance_id) + ec2_id = ec2utils.id_to_ec2_id(instance_id) i['instanceId'] = ec2_id i['imageId'] = instance['image_id'] i['instanceState'] = { @@ -755,7 +745,7 @@ class CloudController(object): if (floating_ip_ref['fixed_ip'] and floating_ip_ref['fixed_ip']['instance']): instance_id = floating_ip_ref['fixed_ip']['instance']['id'] - ec2_id = id_to_ec2_id(instance_id) + ec2_id = ec2utils.id_to_ec2_id(instance_id) address_rv = {'public_ip': address, 'instance_id': ec2_id} if context.is_admin: @@ -778,7 +768,7 @@ class CloudController(object): def associate_address(self, context, instance_id, public_ip, **kwargs): LOG.audit(_("Associate address %(public_ip)s to" " instance %(instance_id)s") % locals(), context=context) - instance_id = ec2_id_to_id(instance_id) + instance_id = ec2utils.ec2_id_to_id(instance_id) self.compute_api.associate_floating_ip(context, instance_id=instance_id, address=public_ip) @@ -791,13 +781,17 @@ class CloudController(object): def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) + if kwargs.get('kernel_id'): + kwargs['kernel_id'] = ec2utils.ec2_id_to_id(kwargs['kernel_id']) + if kwargs.get('ramdisk_id'): + kwargs['ramdisk_id'] = ec2utils.ec2_id_to_id(kwargs['ramdisk_id']) instances = self.compute_api.create(context, instance_type=instance_types.get_by_type( kwargs.get('instance_type', None)), - image_id=kwargs['image_id'], + image_id=ec2utils.ec2_id_to_id(kwargs['image_id']), min_count=int(kwargs.get('min_count', max_count)), max_count=max_count, - kernel_id=kwargs.get('kernel_id', None), + kernel_id=kwargs.get('kernel_id'), ramdisk_id=kwargs.get('ramdisk_id'), display_name=kwargs.get('display_name'), display_description=kwargs.get('display_description'), @@ -814,7 +808,7 @@ class CloudController(object): instance_id is a kwarg so its name cannot be modified.""" LOG.debug(_("Going to start terminating instances")) for ec2_id in instance_id: - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) self.compute_api.delete(context, instance_id=instance_id) return True @@ -822,19 +816,19 @@ class CloudController(object): """instance_id is a list of instance ids""" LOG.audit(_("Reboot instance %r"), instance_id, context=context) for ec2_id in instance_id: - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) self.compute_api.reboot(context, instance_id=instance_id) return True def rescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" - instance_id = ec2_id_to_id(instance_id) + instance_id = ec2utils.ec2_id_to_id(instance_id) self.compute_api.rescue(context, instance_id=instance_id) return True def unrescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" - instance_id = ec2_id_to_id(instance_id) + instance_id = ec2utils.ec2_id_to_id(instance_id) self.compute_api.unrescue(context, instance_id=instance_id) return True @@ -845,41 +839,50 @@ class CloudController(object): if field in kwargs: changes[field] = kwargs[field] if changes: - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) self.compute_api.update(context, instance_id=instance_id, **kwargs) return True - def _format_image(self, context, image): + def _format_image(self, image): """Convert from format defined by BaseImageService to S3 format.""" i = {} i['imageId'] = image.get('id') - i['kernelId'] = image.get('kernel_id') - i['ramdiskId'] = image.get('ramdisk_id') - i['imageOwnerId'] = image.get('owner_id') - i['imageLocation'] = image.get('location') - i['imageState'] = image.get('status') + i['kernelId'] = image['properties'].get('kernel_id') + i['ramdiskId'] = image['properties'].get('ramdisk_id') + i['imageOwnerId'] = image['properties'].get('owner_id') + i['imageLocation'] = image['properties'].get('image_location') + i['imageState'] = image['properties'].get('image_state') i['type'] = image.get('type') - i['isPublic'] = image.get('is_public') - i['architecture'] = image.get('architecture') + i['isPublic'] = image['properties'].get('is_public') == 'True' + i['architecture'] = image['properties'].get('architecture') return i def describe_images(self, context, image_id=None, **kwargs): # NOTE: image_id is a list! - images = self.image_service.index(context) if image_id: - images = filter(lambda x: x['id'] in image_id, images) - images = [self._format_image(context, i) for i in images] + images = [] + for ec2_id in image_id: + try: + image = self.image_service.show(context, ec2_id) + except exception.NotFound: + raise exception.NotFound(_('Image %s not found') % + ec2_id) + images.append(image) + else: + images = self.image_service.detail(context) + images = [self._format_image(i) for i in images] return {'imagesSet': images} def deregister_image(self, context, image_id, **kwargs): LOG.audit(_("De-registering image %s"), image_id, context=context) - self.image_service.deregister(context, image_id) + self.image_service.delete(context, image_id) return {'imageId': image_id} def register_image(self, context, image_location=None, **kwargs): if image_location is None and 'name' in kwargs: image_location = kwargs['name'] - image_id = self.image_service.register(context, image_location) + image = {"image_location": image_location} + image_id = self.image_service.create(context, image) msg = _("Registered image %(image_location)s with" " id %(image_id)s") % locals() LOG.audit(msg, context=context) @@ -890,11 +893,10 @@ class CloudController(object): raise exception.ApiError(_('attribute not supported: %s') % attribute) try: - image = self._format_image(context, - self.image_service.show(context, + image = self._format_image(self.image_service.show(context, image_id)) - except IndexError: - raise exception.ApiError(_('invalid id: %s') % image_id) + except (IndexError, exception.NotFound): + raise exception.NotFound(_('Image %s not found') % image_id) result = {'image_id': image_id, 'launchPermission': []} if image['isPublic']: result['launchPermission'].append({'group': 'all'}) @@ -913,7 +915,14 @@ class CloudController(object): if not operation_type in ['add', 'remove']: raise exception.ApiError(_('operation_type must be add or remove')) LOG.audit(_("Updating image %s publicity"), image_id, context=context) - return self.image_service.modify(context, image_id, operation_type) + + try: + metadata = self.image_service.show(context, image_id) + except exception.NotFound: + raise exception.NotFound(_('Image %s not found') % image_id) + del(metadata['id']) + metadata['properties']['is_public'] = (operation_type == 'add') + return self.image_service.update(context, image_id, metadata) def update_image(self, context, image_id, **kwargs): result = self.image_service.update(context, image_id, dict(kwargs)) diff --git a/nova/flags.py b/nova/flags.py index 8cf199b2f..f01a4d096 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -346,7 +346,7 @@ DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', 'Manager for scheduler') # The service to use for image search and retrieval -DEFINE_string('image_service', 'nova.image.s3.S3ImageService', +DEFINE_string('image_service', 'nova.image.glance.GlanceImageService', 'The service to use for retrieving and searching for images.') DEFINE_string('host', socket.gethostname(), diff --git a/nova/image/glance.py b/nova/image/glance.py index 593c4bce6..7db94c0d4 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -21,6 +21,8 @@ import httplib import json import urlparse +from glance.common import exception as glance_exception + from nova import exception from nova import flags from nova import log as logging @@ -57,27 +59,32 @@ class GlanceImageService(service.BaseImageService): """ Returns a dict containing image data for the given opaque image id. """ - image = self.client.get_image_meta(id) - if image: - return image - raise exception.NotFound + try: + image = self.client.get_image_meta(id) + except glance_exception.NotFound: + raise exception.NotFound + return image - def create(self, context, data): + def create(self, context, metadata, data=None): """ Store the image data and return the new image id. :raises AlreadyExists if the image already exist. """ - return self.client.add_image(image_meta=data) + return self.client.add_image(metadata, data) - def update(self, context, image_id, data): + def update(self, context, image_id, metadata, data=None): """Replace the contents of the given image with the new data. :raises NotFound if the image does not exist. """ - return self.client.update_image(image_id, data) + try: + result = self.client.update_image(image_id, metadata, data) + except glance_exception.NotFound: + raise exception.NotFound + return result def delete(self, context, image_id): """ @@ -86,7 +93,11 @@ class GlanceImageService(service.BaseImageService): :raises NotFound if the image does not exist. """ - return self.client.delete_image(image_id) + try: + result = self.client.delete_image(image_id) + except glance_exception.NotFound: + raise exception.NotFound + return result def delete_all(self): """ diff --git a/nova/image/s3.py b/nova/image/s3.py index 14135a1ee..a740b010c 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -21,8 +21,12 @@ Proxy AMI-related calls from the cloud controller, to the running objectstore service. """ -import json -import urllib +import binascii +import os +import shutil +import tarfile +import tempfile +from xml.etree import ElementTree import boto.s3.connection @@ -31,84 +35,89 @@ from nova import flags from nova import utils from nova.auth import manager from nova.image import service +from nova.api.ec2 import ec2utils FLAGS = flags.FLAGS +flags.DEFINE_string('image_decryption_dir', '/tmp', + 'parent dir for tempdir used for image decryption') -def map_s3_to_base(image): - """Convert from S3 format to format defined by BaseImageService.""" - i = {} - i['id'] = image.get('imageId') - i['name'] = image.get('imageId') - i['kernel_id'] = image.get('kernelId') - i['ramdisk_id'] = image.get('ramdiskId') - i['location'] = image.get('imageLocation') - i['owner_id'] = image.get('imageOwnerId') - i['status'] = image.get('imageState') - i['type'] = image.get('type') - i['is_public'] = image.get('isPublic') - i['architecture'] = image.get('architecture') - return i +_type_prefix_map = {'machine': 'ami', + 'kernel': 'aki', + 'ramdisk': 'ari'} + + +def image_ec2_id(image_id, image_type): + prefix = _type_prefix_map[image_type] + template = prefix + '-%08x' + return ec2utils.id_to_ec2_id(int(image_id), template=template) class S3ImageService(service.BaseImageService): + def __init__(self, service=None, *args, **kwargs): + if service == None: + service = utils.import_object(FLAGS.image_service) + self.service = service + self.service.__init__(*args, **kwargs) + + def create(self, context, properties, data=None): + """image should contain image_location""" + image_id, metadata = self._s3_create(context, properties) + return image_ec2_id(image_id, metadata['type']) + + def delete(self, context, image_id): + # FIXME(vish): call to show is to check filter + self.show(context, image_id) + image_id = ec2utils.ec2_id_to_id(image_id) + self.service.delete(context, image_id) - def modify(self, context, image_id, operation): - self._conn(context).make_request( - method='POST', - bucket='_images', - query_args=self._qs({'image_id': image_id, - 'operation': operation})) - return True - - def update(self, context, image_id, attributes): - """update an image's attributes / info.json""" - attributes.update({"image_id": image_id}) - self._conn(context).make_request( - method='POST', - bucket='_images', - query_args=self._qs(attributes)) - return True - - def register(self, context, image_location): - """ rpc call to register a new image based from a manifest """ - image_id = utils.generate_uid('ami') - self._conn(context).make_request( - method='PUT', - bucket='_images', - query_args=self._qs({'image_location': image_location, - 'image_id': image_id})) - return image_id + def update(self, context, image_id, metadata, data=None): + # FIXME(vish): call to show is to check filter + self.show(context, image_id) + image_id = ec2utils.ec2_id_to_id(image_id) + image = self.service.update(context, image_id, metadata, data) + image['id'] = image_ec2_id(image['id'], image['type']) + return image def index(self, context): - """Return a list of all images that a user can see.""" - response = self._conn(context).make_request( - method='GET', - bucket='_images') - images = json.loads(response.read()) - return [map_s3_to_base(i) for i in images] + images = self.service.index(context) + # FIXME(vish): index doesn't filter so we do it manually + return self._filter(context, images) + + def detail(self, context): + images = self.service.detail(context) + # FIXME(vish): detail doesn't filter so we do it manually + return self._filter(context, images) + + @staticmethod + def _is_visible(context, image): + return (context.is_admin + or context.project_id == image['properties']['owner_id'] + or image['properties']['is_public'] == 'True') + + @staticmethod + def _filter(context, images): + filtered = [] + for image in images: + if not S3ImageService._is_visible(context, image): + continue + image['id'] = image_ec2_id(image['id'], image['type']) + filtered.append(image) + return filtered def show(self, context, image_id): - """return a image object if the context has permissions""" - if FLAGS.connection_type == 'fake': - return {'imageId': 'bar'} - result = self.index(context) - result = [i for i in result if i['id'] == image_id] - if not result: - raise exception.NotFound(_('Image %s could not be found') - % image_id) - image = result[0] + image_id = ec2utils.ec2_id_to_id(image_id) + image = self.service.show(context, image_id) + if not self._is_visible(context, image): + raise exception.NotFound + image['id'] = image_ec2_id(image['id'], image['type']) return image - def deregister(self, context, image_id): - """ unregister an image """ - self._conn(context).make_request( - method='DELETE', - bucket='_images', - query_args=self._qs({'image_id': image_id})) - - def _conn(self, context): + @staticmethod + def _conn(context): + # TODO(vish): is there a better way to get creds to sign + # for the user? access = manager.AuthManager().get_access_key(context.user, context.project) secret = str(context.user.secret) @@ -120,8 +129,139 @@ class S3ImageService(service.BaseImageService): port=FLAGS.s3_port, host=FLAGS.s3_host) - def _qs(self, params): - pairs = [] - for key in params.keys(): - pairs.append(key + '=' + urllib.quote(params[key])) - return '&'.join(pairs) + @staticmethod + def _download_file(bucket, filename, local_dir): + key = bucket.get_key(filename) + local_filename = os.path.join(local_dir, filename) + key.get_contents_to_filename(local_filename) + return local_filename + + def _s3_create(self, context, properties): + image_path = tempfile.mkdtemp(dir=FLAGS.image_decryption_dir) + + image_location = properties['image_location'] + bucket_name = image_location.split("/")[0] + manifest_path = image_location[len(bucket_name) + 1:] + bucket = self._conn(context).get_bucket(bucket_name) + key = bucket.get_key(manifest_path) + manifest = key.get_contents_as_string() + + manifest = ElementTree.fromstring(manifest) + image_type = 'machine' + + try: + kernel_id = manifest.find("machine_configuration/kernel_id").text + if kernel_id == 'true': + image_type = 'kernel' + kernel_id = None + except: + kernel_id = None + + try: + ramdisk_id = manifest.find("machine_configuration/ramdisk_id").text + if ramdisk_id == 'true': + image_type = 'ramdisk' + ramdisk_id = None + except: + ramdisk_id = None + + try: + arch = manifest.find("machine_configuration/architecture").text + except: + arch = 'x86_64' + + properties.update({'owner_id': context.project_id, + 'architecture': arch}) + + if kernel_id: + properties['kernel_id'] = kernel_id + + if ramdisk_id: + properties['ramdisk_id'] = ramdisk_id + + properties['is_public'] = False + metadata = {'type': image_type, + 'status': 'queued', + 'is_public': True, + 'properties': properties} + metadata['properties']['image_state'] = 'pending' + image = self.service.create(context, metadata) + image_id = image['id'] + + parts = [] + for fn_element in manifest.find("image").getiterator("filename"): + part = self._download_file(bucket, fn_element.text, image_path) + parts.append(part) + + # NOTE(vish): this may be suboptimal, should we use cat? + encrypted_filename = os.path.join(image_path, 'image.encrypted') + with open(encrypted_filename, 'w') as combined: + for filename in parts: + with open(filename) as part: + shutil.copyfileobj(part, combined) + + metadata['properties']['image_state'] = 'decrypting' + self.service.update(context, image_id, metadata) + + hex_key = manifest.find("image/ec2_encrypted_key").text + encrypted_key = binascii.a2b_hex(hex_key) + hex_iv = manifest.find("image/ec2_encrypted_iv").text + encrypted_iv = binascii.a2b_hex(hex_iv) + + # FIXME(vish): grab key from common service so this can run on + # any host. + cloud_private_key = os.path.join(FLAGS.ca_path, "private/cakey.pem") + + decrypted_filename = os.path.join(image_path, 'image.tar.gz') + self._decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, + cloud_private_key, decrypted_filename) + + metadata['properties']['image_state'] = 'untarring' + self.service.update(context, image_id, metadata) + + unz_filename = self._untarzip_image(image_path, decrypted_filename) + + metadata['properties']['image_state'] = 'uploading' + with open(unz_filename) as image_file: + self.service.update(context, image_id, metadata, image_file) + metadata['properties']['image_state'] = 'available' + self.service.update(context, image_id, metadata) + + shutil.rmtree(image_path) + return image_id, metadata + + @staticmethod + def _decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, + cloud_private_key, decrypted_filename): + key, err = utils.execute( + 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, + process_input=encrypted_key, + check_exit_code=False) + if err: + raise exception.Error(_("Failed to decrypt private key: %s") + % err) + iv, err = utils.execute( + 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, + process_input=encrypted_iv, + check_exit_code=False) + if err: + raise exception.Error(_("Failed to decrypt initialization " + "vector: %s") % err) + + _out, err = utils.execute( + 'openssl enc -d -aes-128-cbc -in %s -K %s -iv %s -out %s' + % (encrypted_filename, key, iv, decrypted_filename), + check_exit_code=False) + if err: + raise exception.Error(_("Failed to decrypt image file " + "%(image_file)s: %(err)s") % + {'image_file': encrypted_filename, + 'err': err}) + + @staticmethod + def _untarzip_image(path, filename): + tar_file = tarfile.open(filename, "r|gz") + tar_file.extractall(path) + image_file = tar_file.getnames()[0] + tar_file.close() + return os.path.join(path, image_file) diff --git a/nova/image/service.py b/nova/image/service.py index ebee2228d..e429955f4 100644 --- a/nova/image/service.py +++ b/nova/image/service.py @@ -76,7 +76,7 @@ class BaseImageService(object): """ raise NotImplementedError - def create(self, context, data): + def create(self, context, metadata, data=None): """ Store the image data and return the new image id. @@ -85,7 +85,7 @@ class BaseImageService(object): """ raise NotImplementedError - def update(self, context, image_id, data): + def update(self, context, image_id, metadata, data=None): """Replace the contents of the given image with the new data. :raises NotFound if the image does not exist. diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 49ce8c1b5..7b016db08 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -25,6 +25,7 @@ import webob.dec from paste import urlmap from glance import client as glance_client +from glance.common import exception as glance_exc from nova import auth from nova import context @@ -149,25 +150,25 @@ def stub_out_glance(stubs, initial_fixtures=None): for f in self.fixtures: if f['id'] == image_id: return f - return None + raise glance_exc.NotFound - def fake_add_image(self, image_meta): + def fake_add_image(self, image_meta, data=None): id = ''.join(random.choice(string.letters) for _ in range(20)) image_meta['id'] = id self.fixtures.append(image_meta) return id - def fake_update_image(self, image_id, image_meta): + def fake_update_image(self, image_id, image_meta, data=None): f = self.fake_get_image_meta(image_id) if not f: - raise exc.NotFound + raise glance_exc.NotFound f.update(image_meta) def fake_delete_image(self, image_id): f = self.fake_get_image_meta(image_id) if not f: - raise exc.NotFound + raise glance_exc.NotFound self.fixtures.remove(f) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index cbd949477..5d7ca98b5 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -32,6 +32,7 @@ flags.DECLARE('fake_network', 'nova.network.manager') FLAGS.network_size = 8 FLAGS.num_networks = 2 FLAGS.fake_network = True +FLAGS.image_service = 'nova.image.local.LocalImageService' flags.DECLARE('num_shelves', 'nova.volume.driver') flags.DECLARE('blades_per_shelf', 'nova.volume.driver') flags.DECLARE('iscsi_num_targets', 'nova.volume.driver') diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 061910013..7d7b91658 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -38,6 +38,8 @@ from nova import test from nova.auth import manager from nova.compute import power_state from nova.api.ec2 import cloud +from nova.api.ec2 import ec2utils +from nova.image import local from nova.objectstore import image @@ -76,6 +78,11 @@ class CloudTestCase(test.TestCase): project=self.project) host = self.network.get_network_host(self.context.elevated()) + def fake_image_show(meh, context, id): + return dict(kernelId=1, ramdiskId=1) + + self.stubs.Set(local.LocalImageService, 'show', fake_image_show) + def tearDown(self): network_ref = db.project_get_network(self.context, self.project.id) @@ -122,7 +129,7 @@ class CloudTestCase(test.TestCase): self.cloud.allocate_address(self.context) inst = db.instance_create(self.context, {'host': self.compute.host}) fixed = self.network.allocate_fixed_ip(self.context, inst['id']) - ec2_id = cloud.id_to_ec2_id(inst['id']) + ec2_id = ec2utils.id_to_ec2_id(inst['id']) self.cloud.associate_address(self.context, instance_id=ec2_id, public_ip=address) @@ -158,12 +165,12 @@ class CloudTestCase(test.TestCase): vol2 = db.volume_create(self.context, {}) result = self.cloud.describe_volumes(self.context) self.assertEqual(len(result['volumeSet']), 2) - volume_id = cloud.id_to_ec2_id(vol2['id'], 'vol-%08x') + volume_id = ec2utils.id_to_ec2_id(vol2['id'], 'vol-%08x') result = self.cloud.describe_volumes(self.context, volume_id=[volume_id]) self.assertEqual(len(result['volumeSet']), 1) self.assertEqual( - cloud.ec2_id_to_id(result['volumeSet'][0]['volumeId']), + ec2utils.ec2_id_to_id(result['volumeSet'][0]['volumeId']), vol2['id']) db.volume_destroy(self.context, vol1['id']) db.volume_destroy(self.context, vol2['id']) @@ -200,7 +207,7 @@ class CloudTestCase(test.TestCase): result = self.cloud.describe_instances(self.context) result = result['reservationSet'][0] self.assertEqual(len(result['instancesSet']), 2) - instance_id = cloud.id_to_ec2_id(inst2['id']) + instance_id = ec2utils.id_to_ec2_id(inst2['id']) result = self.cloud.describe_instances(self.context, instance_id=[instance_id]) result = result['reservationSet'][0] @@ -216,6 +223,7 @@ class CloudTestCase(test.TestCase): def test_console_output(self): image_id = FLAGS.default_image + print image_id instance_type = FLAGS.default_instance_type max_count = 1 kwargs = {'image_id': image_id, @@ -347,7 +355,7 @@ class CloudTestCase(test.TestCase): def test_update_of_instance_display_fields(self): inst = db.instance_create(self.context, {}) - ec2_id = cloud.id_to_ec2_id(inst['id']) + ec2_id = ec2utils.id_to_ec2_id(inst['id']) self.cloud.update_instance(self.context, ec2_id, display_name='c00l 1m4g3') inst = db.instance_get(self.context, inst['id']) @@ -365,7 +373,7 @@ class CloudTestCase(test.TestCase): def test_update_of_volume_display_fields(self): vol = db.volume_create(self.context, {}) self.cloud.update_volume(self.context, - cloud.id_to_ec2_id(vol['id'], 'vol-%08x'), + ec2utils.id_to_ec2_id(vol['id'], 'vol-%08x'), display_name='c00l v0lum3') vol = db.volume_get(self.context, vol['id']) self.assertEqual('c00l v0lum3', vol['display_name']) @@ -374,7 +382,7 @@ class CloudTestCase(test.TestCase): def test_update_of_volume_wont_update_private_fields(self): vol = db.volume_create(self.context, {}) self.cloud.update_volume(self.context, - cloud.id_to_ec2_id(vol['id'], 'vol-%08x'), + ec2utils.id_to_ec2_id(vol['id'], 'vol-%08x'), mountpoint='/not/here') vol = db.volume_get(self.context, vol['id']) self.assertEqual(None, vol['mountpoint']) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 949b5e6eb..1f49baaf6 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -31,7 +31,7 @@ from nova import test from nova import utils from nova.auth import manager from nova.compute import instance_types - +from nova.image import local LOG = logging.getLogger('nova.tests.compute') FLAGS = flags.FLAGS @@ -47,6 +47,11 @@ class ComputeTestCase(test.TestCase): network_manager='nova.network.manager.FlatManager') self.compute = utils.import_object(FLAGS.compute_manager) self.compute_api = compute.API() + + def fake_image_show(meh, context, id): + return dict(kernelId=1, ramdiskId=1) + + self.stubs.Set(local.LocalImageService, 'show', fake_image_show) self.manager = manager.AuthManager() self.user = self.manager.create_user('fake', 'fake', 'fake') self.project = self.manager.create_project('fake', 'fake', 'fake') diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index b6bfab534..b130e3f53 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -90,8 +90,7 @@ class DirectTestCase(test.TestCase): class DirectCloudTestCase(test_cloud.CloudTestCase): def setUp(self): super(DirectCloudTestCase, self).setUp() - compute_handle = compute.API(image_service=self.cloud.image_service, - network_api=self.cloud.network_api, + compute_handle = compute.API(network_api=self.cloud.network_api, volume_api=self.cloud.volume_api) direct.register_service('compute', compute_handle) self.router = direct.JsonParamsMiddleware(direct.Router()) diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 4ecb36b54..ca8abdb36 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -57,7 +57,7 @@ class QuotaTestCase(test.TestCase): def _create_instance(self, cores=2): """Create a test instance""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 'ami-1' inst['reservation_id'] = 'r-fakeres' inst['user_id'] = self.user.id inst['project_id'] = self.project.id @@ -123,7 +123,7 @@ class QuotaTestCase(test.TestCase): min_count=1, max_count=1, instance_type='m1.small', - image_id='fake') + image_id='ami-1') for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -136,7 +136,7 @@ class QuotaTestCase(test.TestCase): min_count=1, max_count=1, instance_type='m1.small', - image_id='fake') + image_id='ami-1') for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) -- cgit From bc94ec23100de9f07e04b0348823d4f103a9daa5 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 4 Mar 2011 02:49:12 +0000 Subject: use LocalImageServiceByDefault --- nova/flags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/flags.py b/nova/flags.py index f01a4d096..cb47ca8d1 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -346,7 +346,7 @@ DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', 'Manager for scheduler') # The service to use for image search and retrieval -DEFINE_string('image_service', 'nova.image.glance.GlanceImageService', +DEFINE_string('image_service', 'nova.image.local.LocalImageService', 'The service to use for retrieving and searching for images.') DEFINE_string('host', socket.gethostname(), -- cgit From 13307e02258a5a08bedb1ed933a107668aac6457 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 4 Mar 2011 05:04:49 +0000 Subject: make local image service work --- nova/api/ec2/cloud.py | 2 +- nova/image/local.py | 64 ++++++++++++++++++++++++++++++++--------------- nova/objectstore/image.py | 3 +-- 3 files changed, 46 insertions(+), 23 deletions(-) (limited to 'nova') diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 8c2e77d86..aa1dcbe33 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -853,7 +853,7 @@ class CloudController(object): i['imageLocation'] = image['properties'].get('image_location') i['imageState'] = image['properties'].get('image_state') i['type'] = image.get('type') - i['isPublic'] = image['properties'].get('is_public') == 'True' + i['isPublic'] = str(image['properties'].get('is_public', '')) == 'True' i['architecture'] = image['properties'].get('architecture') return i diff --git a/nova/image/local.py b/nova/image/local.py index f78b9aa89..b4616729a 100644 --- a/nova/image/local.py +++ b/nova/image/local.py @@ -15,57 +15,81 @@ # License for the specific language governing permissions and limitations # under the License. -import cPickle as pickle +import json import os.path import random -import tempfile +import shutil +from nova import flags from nova import exception from nova.image import service -class LocalImageService(service.BaseImageService): +FLAGS = flags.FLAGS +flags.DEFINE_string('images_path', '$state_path/images', + 'path to decrypted images') +class LocalImageService(service.BaseImageService): """Image service storing images to local disk. + It assumes that image_ids are integers. """ def __init__(self): - self._path = tempfile.mkdtemp() + self._path = FLAGS.images_path - def _path_to(self, image_id): + def _path_to(self, image_id, fname='info.json'): + if fname: + return os.path.join(self._path, str(image_id), fname) return os.path.join(self._path, str(image_id)) def _ids(self): """The list of all image ids.""" - return [int(i) for i in os.listdir(self._path)] + return [int(i) for i in os.listdir(self._path) + if unicode(i).isnumeric()] def index(self, context): - return [dict(id=i['id'], name=i['name']) for i in self.detail(context)] + return [dict(image_id=i['id'], name=i.get('name')) + for i in self.detail(context)] def detail(self, context): - return [self.show(context, id) for id in self._ids()] - - def show(self, context, id): + images = [] + for image_id in self._ids(): + try: + image = self.show(context, image_id) + images.append(image) + except exception.NotFound: + continue + return images + + def show(self, context, image_id): try: - return pickle.load(open(self._path_to(id))) + with open(self._path_to(image_id)) as metadata_file: + return json.load(metadata_file) except IOError: raise exception.NotFound - def create(self, context, data): + def create(self, context, metadata, data=None): """Store the image data and return the new image id.""" - id = random.randint(0, 2 ** 31 - 1) - data['id'] = id - self.update(context, id, data) - return id + image_id = random.randint(0, 2 ** 31 - 1) + image_path = self._path_to(image_id, None) + if not os.path.exists(image_path): + os.mkdir(image_path) + return self.update(context, image_id, metadata, data) - def update(self, context, image_id, data): + def update(self, context, image_id, metadata, data=None): """Replace the contents of the given image with the new data.""" + metadata['id'] = image_id try: - pickle.dump(data, open(self._path_to(image_id), 'w')) + with open(self._path_to(image_id), 'w') as metadata_file: + json.dump(metadata, metadata_file) + if data: + with open(self._path_to(image_id, 'image'), 'w') as image_file: + shutil.copyfileobj(data, image_file) except IOError: raise exception.NotFound + return metadata def delete(self, context, image_id): """Delete the given image. @@ -79,8 +103,8 @@ class LocalImageService(service.BaseImageService): def delete_all(self): """Clears out all images in local directory.""" - for id in self._ids(): - os.unlink(self._path_to(id)) + for image_id in self._ids(): + os.unlink(self._path_to(image_id)) def delete_imagedir(self): """Deletes the local directory. diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index 27227e2ca..8013cbd9c 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -37,8 +37,7 @@ from nova.objectstore import bucket FLAGS = flags.FLAGS -flags.DEFINE_string('images_path', '$state_path/images', - 'path to decrypted images') +flags.DECLARE('images_path', 'nova.image.local') class Image(object): -- cgit From cf9bc248f0fc318c4a9fb5087f257216312e39d1 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 4 Mar 2011 05:21:28 +0000 Subject: fix a couple issues with local, update the glance fake to actually return the same types as the real client, fix the image tests --- nova/image/local.py | 12 +++--------- nova/tests/api/openstack/fakes.py | 3 ++- nova/tests/api/openstack/test_images.py | 15 +++++++++------ 3 files changed, 14 insertions(+), 16 deletions(-) (limited to 'nova') diff --git a/nova/image/local.py b/nova/image/local.py index b4616729a..6fa648b6b 100644 --- a/nova/image/local.py +++ b/nova/image/local.py @@ -29,6 +29,7 @@ FLAGS = flags.FLAGS flags.DEFINE_string('images_path', '$state_path/images', 'path to decrypted images') + class LocalImageService(service.BaseImageService): """Image service storing images to local disk. @@ -97,18 +98,11 @@ class LocalImageService(service.BaseImageService): """ try: - os.unlink(self._path_to(image_id)) + shutil.rmtree(self._path_to(image_id, None)) except IOError: raise exception.NotFound def delete_all(self): """Clears out all images in local directory.""" for image_id in self._ids(): - os.unlink(self._path_to(image_id)) - - def delete_imagedir(self): - """Deletes the local directory. - Raises OSError if directory is not empty. - - """ - os.rmdir(self._path) + shutil.rmtree(self._path_to(image_id, None)) diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 7b016db08..2c4e57246 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -156,7 +156,7 @@ def stub_out_glance(stubs, initial_fixtures=None): id = ''.join(random.choice(string.letters) for _ in range(20)) image_meta['id'] = id self.fixtures.append(image_meta) - return id + return image_meta def fake_update_image(self, image_id, image_meta, data=None): f = self.fake_get_image_meta(image_id) @@ -164,6 +164,7 @@ def stub_out_glance(stubs, initial_fixtures=None): raise glance_exc.NotFound f.update(image_meta) + return f def fake_delete_image(self, image_id): f = self.fake_get_image_meta(image_id) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index e232bc3d5..eb5039bdb 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -22,6 +22,8 @@ and as a WSGI layer import json import datetime +import shutil +import tempfile import stubout import webob @@ -54,7 +56,7 @@ class BaseImageServiceTests(object): num_images = len(self.service.index(self.context)) - id = self.service.create(self.context, fixture) + id = self.service.create(self.context, fixture)['id'] self.assertNotEquals(None, id) self.assertEquals(num_images + 1, @@ -71,7 +73,7 @@ class BaseImageServiceTests(object): num_images = len(self.service.index(self.context)) - id = self.service.create(self.context, fixture) + id = self.service.create(self.context, fixture)['id'] self.assertNotEquals(None, id) @@ -89,7 +91,7 @@ class BaseImageServiceTests(object): 'instance_id': None, 'progress': None} - id = self.service.create(self.context, fixture) + id = self.service.create(self.context, fixture)['id'] fixture['status'] = 'in progress' @@ -118,7 +120,7 @@ class BaseImageServiceTests(object): ids = [] for fixture in fixtures: - new_id = self.service.create(self.context, fixture) + new_id = self.service.create(self.context, fixture)['id'] ids.append(new_id) num_images = len(self.service.index(self.context)) @@ -137,14 +139,15 @@ class LocalImageServiceTest(test.TestCase, def setUp(self): super(LocalImageServiceTest, self).setUp() + self.tempdir = tempfile.mkdtemp() + self.flags(images_path=self.tempdir) self.stubs = stubout.StubOutForTesting() service_class = 'nova.image.local.LocalImageService' self.service = utils.import_object(service_class) self.context = context.RequestContext(None, None) def tearDown(self): - self.service.delete_all() - self.service.delete_imagedir() + shutil.rmtree(self.tempdir) self.stubs.UnsetAll() super(LocalImageServiceTest, self).tearDown() -- cgit From e2c95e198f1982bc50bc95bc61ef3211b17937a2 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 4 Mar 2011 05:55:41 +0000 Subject: spawn a greenthread for image registration because it is slow --- nova/image/s3.py | 67 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 36 insertions(+), 31 deletions(-) (limited to 'nova') diff --git a/nova/image/s3.py b/nova/image/s3.py index a740b010c..e9542c7bd 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -22,6 +22,7 @@ objectstore service. """ import binascii +import eventlet import os import shutil import tarfile @@ -188,46 +189,50 @@ class S3ImageService(service.BaseImageService): image = self.service.create(context, metadata) image_id = image['id'] - parts = [] - for fn_element in manifest.find("image").getiterator("filename"): - part = self._download_file(bucket, fn_element.text, image_path) - parts.append(part) + def delayed_create(): + parts = [] + for fn_element in manifest.find("image").getiterator("filename"): + part = self._download_file(bucket, fn_element.text, image_path) + parts.append(part) - # NOTE(vish): this may be suboptimal, should we use cat? - encrypted_filename = os.path.join(image_path, 'image.encrypted') - with open(encrypted_filename, 'w') as combined: - for filename in parts: - with open(filename) as part: - shutil.copyfileobj(part, combined) + # NOTE(vish): this may be suboptimal, should we use cat? + encrypted_filename = os.path.join(image_path, 'image.encrypted') + with open(encrypted_filename, 'w') as combined: + for filename in parts: + with open(filename) as part: + shutil.copyfileobj(part, combined) - metadata['properties']['image_state'] = 'decrypting' - self.service.update(context, image_id, metadata) + metadata['properties']['image_state'] = 'decrypting' + self.service.update(context, image_id, metadata) - hex_key = manifest.find("image/ec2_encrypted_key").text - encrypted_key = binascii.a2b_hex(hex_key) - hex_iv = manifest.find("image/ec2_encrypted_iv").text - encrypted_iv = binascii.a2b_hex(hex_iv) + hex_key = manifest.find("image/ec2_encrypted_key").text + encrypted_key = binascii.a2b_hex(hex_key) + hex_iv = manifest.find("image/ec2_encrypted_iv").text + encrypted_iv = binascii.a2b_hex(hex_iv) - # FIXME(vish): grab key from common service so this can run on - # any host. - cloud_private_key = os.path.join(FLAGS.ca_path, "private/cakey.pem") + # FIXME(vish): grab key from common service so this can run on + # any host. + cloud_pk = os.path.join(FLAGS.ca_path, "private/cakey.pem") - decrypted_filename = os.path.join(image_path, 'image.tar.gz') - self._decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, - cloud_private_key, decrypted_filename) + decrypted_filename = os.path.join(image_path, 'image.tar.gz') + self._decrypt_image(encrypted_filename, encrypted_key, + encrypted_iv, cloud_pk, decrypted_filename) - metadata['properties']['image_state'] = 'untarring' - self.service.update(context, image_id, metadata) + metadata['properties']['image_state'] = 'untarring' + self.service.update(context, image_id, metadata) - unz_filename = self._untarzip_image(image_path, decrypted_filename) + unz_filename = self._untarzip_image(image_path, decrypted_filename) - metadata['properties']['image_state'] = 'uploading' - with open(unz_filename) as image_file: - self.service.update(context, image_id, metadata, image_file) - metadata['properties']['image_state'] = 'available' - self.service.update(context, image_id, metadata) + metadata['properties']['image_state'] = 'uploading' + with open(unz_filename) as image_file: + self.service.update(context, image_id, metadata, image_file) + metadata['properties']['image_state'] = 'available' + self.service.update(context, image_id, metadata) + + shutil.rmtree(image_path) + + eventlet.spawn_n(delayed_create) - shutil.rmtree(image_path) return image_id, metadata @staticmethod -- cgit From 517a571f8905c32efd45f7b3410fb263ad705545 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 4 Mar 2011 05:58:49 +0000 Subject: add the ec2utils file i forgot --- nova/api/ec2/ec2utils.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 nova/api/ec2/ec2utils.py (limited to 'nova') diff --git a/nova/api/ec2/ec2utils.py b/nova/api/ec2/ec2utils.py new file mode 100644 index 000000000..0ea22c0e6 --- /dev/null +++ b/nova/api/ec2/ec2utils.py @@ -0,0 +1,27 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def ec2_id_to_id(ec2_id): + """Convert an ec2 ID (i-[base 16 number]) to an instance id (int)""" + return int(ec2_id.split('-')[-1], 16) + + +def id_to_ec2_id(instance_id, template='i-%08x'): + """Convert an instance ID (int) to an ec2 ID (i-[base 16 number])""" + return template % instance_id -- cgit From abd5779068f3b979fc79dec7a68549999c58092d Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Fri, 4 Mar 2011 01:36:29 -0500 Subject: remove ensure_b64_encoding --- nova/compute/manager.py | 10 ++-------- nova/utils.py | 12 ------------ nova/virt/xenapi/vmops.py | 10 +++++----- 3 files changed, 7 insertions(+), 25 deletions(-) (limited to 'nova') diff --git a/nova/compute/manager.py b/nova/compute/manager.py index d915dc069..3a712fd97 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -34,7 +34,6 @@ terminating it. :func:`nova.utils.import_object` """ -import base64 import datetime import random import string @@ -353,15 +352,10 @@ class ComputeManager(manager.Manager): LOG.warn(_('trying to inject a file into a non-running ' 'instance: %(instance_id)s (state: %(instance_state)s ' 'expected: %(expected_state)s)') % locals()) - # Files/paths *should* be base64-encoded at this point, but - # double-check to make sure. - b64_path = utils.ensure_b64_encoding(path) - b64_contents = utils.ensure_b64_encoding(file_contents) - plain_path = base64.b64decode(b64_path) nm = instance_ref['name'] - msg = _('instance %(nm)s: injecting file to %(plain_path)s') % locals() + msg = _('instance %(nm)s: injecting file to %(path)s') % locals() LOG.audit(msg) - self.driver.inject_file(instance_ref, b64_path, b64_contents) + self.driver.inject_file(instance_ref, path, file_contents) @exception.wrap_exception @checks_instance_lock diff --git a/nova/utils.py b/nova/utils.py index 0cf91e0cc..02b71900c 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -491,18 +491,6 @@ def loads(s): return json.loads(s) -def ensure_b64_encoding(val): - """Safety method to ensure that values expected to be base64-encoded - actually are. If they are, the value is returned unchanged. Otherwise, - the encoded value is returned. - """ - try: - dummy = base64.decode(val) - return val - except TypeError: - return base64.b64encode(val) - - def get_from_path(items, path): """ Returns a list of items matching the specified path. Takes an XPath-like expression e.g. prop1/prop2/prop3, and for each item in items, diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 9ac83efb0..89d58a664 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -19,6 +19,7 @@ Management class for VM-related functions (spawn, reboot, etc). """ +import base64 import json import M2Crypto import os @@ -313,17 +314,16 @@ class VMOps(object): task = self._session.call_xenapi("Async.VM.start", vm, False, False) self._session.wait_for_task(task, instance.id) - def inject_file(self, instance, b64_path, b64_contents): + def inject_file(self, instance, path, contents): """Write a file to the VM instance. The path to which it is to be written and the contents of the file need to be supplied; both should be base64-encoded to prevent errors with non-ASCII characters being transmitted. If the agent does not support file injection, or the user has disabled it, a NotImplementedError will be raised. """ - # Files/paths *should* be base64-encoded at this point, but - # double-check to make sure. - b64_path = utils.ensure_b64_encoding(b64_path) - b64_contents = utils.ensure_b64_encoding(b64_contents) + # Files/paths must be base64-encoded for transmission to agent + b64_path = base64.b64encode(path) + b64_contents = base64.b64encode(contents) # Need to uniquely identify this request. transaction_id = str(uuid.uuid4()) -- cgit From 1f0df07baac52379b122a9928200305dd9d2151f Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Sat, 5 Mar 2011 00:57:08 +0900 Subject: Fixed based on reviewer's comment. Main changes are below. 1. get_vcpu_total()/get_memory_mb()/get_memory_mb_used() is changed for users who used non-linux environment. 2. test code added to test_virt. --- nova/tests/test_virt.py | 163 ++++++++++++++++++++++++++++++++++++++-------- nova/virt/libvirt_conn.py | 12 +++- 2 files changed, 146 insertions(+), 29 deletions(-) (limited to 'nova') diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index aac55a894..5bb31659b 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -15,6 +15,7 @@ # under the License. import mox +import sys from xml.etree.ElementTree import fromstring as xml_to_tree from xml.dom.minidom import parseString as xml_to_dom @@ -27,11 +28,15 @@ from nova import test from nova import utils from nova.api.ec2 import cloud from nova.auth import manager +from nova.compute import manager as compute_manager +from nova.compute import power_state from nova.db.sqlalchemy import models from nova.virt import libvirt_conn +libvirt = None FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') +flags.DECLARE('compute_driver', 'nova.compute.manager') class LibvirtConnTestCase(test.TestCase): @@ -73,31 +78,36 @@ class LibvirtConnTestCase(test.TestCase): 'bridge': 'br101', 'instance_type': 'm1.small'} + def lazy_load_library_exists(self): + """check if libvirt is available.""" + # try to connect libvirt. if fail, skip test. + try: + import libvirt + import libxml2 + except ImportError: + return False + global libvirt + libvirt = __import__('libvirt') + libvirt_conn.libvirt = __import__('libvirt') + libvirt_conn.libxml2 = __import__('libxml2') + return True + def create_fake_libvirt_mock(self, **kwargs): """Defining mocks for LibvirtConnection(libvirt is not used).""" - # A fake libvirt.virtConnect + # A fake libvirt.virConnect class FakeLibvirtConnection(object): - def getVersion(self): - return 12003 - - def getType(self): - return 'qemu' - - def getCapabilities(self): - return 'qemu' - - def listDomainsID(self): - return [] - - def getCapabilitied(self): - return + pass # A fake libvirt_conn.IptablesFirewallDriver class FakeIptablesFirewallDriver(object): + def __init__(self, **kwargs): pass + def setattr(self, key, val): + self.__setattr__(key, val) + # Creating mocks fake = FakeLibvirtConnection() fakeip = FakeIptablesFirewallDriver @@ -274,33 +284,54 @@ class LibvirtConnTestCase(test.TestCase): self.assertEquals(uri, testuri) db.instance_destroy(user_context, instance_ref['id']) - def test_update_available_resource_works_correctly(self): + def tes1t_update_available_resource_works_correctly(self): """Confirm compute_node table is updated successfully.""" org_path = FLAGS.instances_path = '' FLAGS.instances_path = '.' + # Prepare mocks + def getVersion(): + return 12003 + + def getType(): + return 'qemu' + + def listDomainsID(): + return [] + service_ref = self.create_service(host='dummy') - self.create_fake_libvirt_mock() + self.create_fake_libvirt_mock(getVersion=getVersion, + getType=getType, + listDomainsID=listDomainsID) self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, 'get_cpu_info') libvirt_conn.LibvirtConnection.get_cpu_info().AndReturn('cpuinfo') + # Start test self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) conn.update_available_resource(self.context, 'dummy') service_ref = db.service_get(self.context, service_ref['id']) compute_node = service_ref['compute_node'][0] - c1 = (compute_node['vcpus'] > 0) - c2 = (compute_node['memory_mb'] > 0) - c3 = (compute_node['local_gb'] > 0) - c4 = (compute_node['vcpus_used'] == 0) - c5 = (compute_node['memory_mb_used'] > 0) - c6 = (compute_node['local_gb_used'] > 0) - c7 = (len(compute_node['hypervisor_type']) > 0) - c8 = (compute_node['hypervisor_version'] > 0) - - self.assertTrue(c1 and c2 and c3 and c4 and c5 and c6 and c7 and c8) + if sys.platform.upper() == 'LINUX2': + self.assertTrue(compute_node['vcpus'] > 0) + self.assertTrue(compute_node['memory_mb'] > 0) + self.assertTrue(compute_node['local_gb'] > 0) + self.assertTrue(compute_node['vcpus_used'] == 0) + self.assertTrue(compute_node['memory_mb_used'] > 0) + self.assertTrue(compute_node['local_gb_used'] > 0) + self.assertTrue(len(compute_node['hypervisor_type']) > 0) + self.assertTrue(compute_node['hypervisor_version'] > 0) + else: + self.assertTrue(compute_node['vcpus'] > 0) + self.assertTrue(compute_node['memory_mb'] == 0) + self.assertTrue(compute_node['local_gb'] > 0) + self.assertTrue(compute_node['vcpus_used'] == 0) + self.assertTrue(compute_node['memory_mb_used'] == 0) + self.assertTrue(compute_node['local_gb_used'] > 0) + self.assertTrue(len(compute_node['hypervisor_type']) > 0) + self.assertTrue(compute_node['hypervisor_version'] > 0) db.service_destroy(self.context, service_ref['id']) FLAGS.instances_path = org_path @@ -319,6 +350,84 @@ class LibvirtConnTestCase(test.TestCase): FLAGS.instances_path = org_path + def test_ensure_filtering_rules_for_instance_timeout(self): + """ensure_filtering_fules_for_instance() finishes with timeout.""" + # Skip if non-libvirt environment + if not self.lazy_load_library_exists(): + return + + # Preparing mocks + def fake_none(self): + return + + def fake_raise(self): + raise libvirt.libvirtError('ERR') + + self.create_fake_libvirt_mock(nwfilterLookupByName=fake_raise) + instance_ref = db.instance_create(self.context, self.test_instance) + + # Start test + self.mox.ReplayAll() + try: + conn = libvirt_conn.LibvirtConnection(False) + conn.firewall_driver.setattr('setup_basic_filtering', fake_none) + conn.firewall_driver.setattr('prepare_instance_filter', fake_none) + conn.ensure_filtering_rules_for_instance(instance_ref) + except exception.Error, e: + c1 = (0 <= e.message.find('Timeout migrating for')) + self.assertTrue(c1) + + db.instance_destroy(self.context, instance_ref['id']) + + def test_live_migration_raises_exception(self): + """Confirms recover method is called when exceptions are raised.""" + # Skip if non-libvirt environment + if not self.lazy_load_library_exists(): + return + + # Preparing data + self.compute = utils.import_object(FLAGS.compute_manager) + instance_dict = {'host': 'fake', 'state': power_state.RUNNING, + 'state_description': 'running'} + instance_ref = db.instance_create(self.context, self.test_instance) + instance_ref = db.instance_update(self.context, instance_ref['id'], + instance_dict) + vol_dict = {'status': 'migrating', 'size': 1} + volume_ref = db.volume_create(self.context, vol_dict) + db.volume_attached(self.context, volume_ref['id'], instance_ref['id'], + '/dev/fake') + + # Preparing mocks + vdmock = self.mox.CreateMock(libvirt.virDomain) + self.mox.StubOutWithMock(vdmock, "migrateToURI") + vdmock.migrateToURI(FLAGS.live_migration_uri % 'dest', + mox.IgnoreArg(), + None, FLAGS.live_migration_bandwidth).\ + AndRaise(libvirt.libvirtError('ERR')) + + def fake_lookup(instance_name): + if instance_name == instance_ref.name: + return vdmock + + self.create_fake_libvirt_mock(lookupByName=fake_lookup) + + # Start test + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertRaises(libvirt.libvirtError, + conn._live_migration, + self.context, instance_ref, 'dest', '', + self.compute.recover_live_migration) + + instance_ref = db.instance_get(self.context, instance_ref['id']) + self.assertTrue(instance_ref['state_description'] == 'running') + self.assertTrue(instance_ref['state'] == power_state.RUNNING) + volume_ref = db.volume_get(self.context, volume_ref['id']) + self.assertTrue(volume_ref['status'] == 'in-use') + + db.volume_destroy(self.context, volume_ref['id']) + db.instance_destroy(self.context, instance_ref['id']) + def tearDown(self): self.manager.delete_project(self.project) self.manager.delete_user(self.user) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 71ca508b0..627a12a1c 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -36,8 +36,10 @@ Supports KVM, QEMU, UML, and XEN. """ +import multiprocessing import os import shutil +import sys import random import subprocess import time @@ -858,7 +860,7 @@ class LibvirtConnection(object): """ - return open('/proc/cpuinfo').read().count('processor') + return multiprocessing.cpu_count() def get_memory_mb_total(self): """Get the total memory size(MB) of physical computer. @@ -867,6 +869,9 @@ class LibvirtConnection(object): """ + if sys.platform.upper() != 'LINUX2': + return 0 + meminfo = open('/proc/meminfo').read().split() idx = meminfo.index('MemTotal:') # transforming kb to mb. @@ -905,6 +910,9 @@ class LibvirtConnection(object): """ + if sys.platform.upper() != 'LINUX2': + return 0 + m = open('/proc/meminfo').read().split() idx1 = m.index('MemFree:') idx2 = m.index('Buffers:') @@ -1126,7 +1134,7 @@ class LibvirtConnection(object): # wait for completion timeout_count = range(FLAGS.live_migration_retry_count) - while not timeout_count: + while timeout_count: try: filter_name = 'nova-instance-%s' % instance_ref.name self._conn.nwfilterLookupByName(filter_name) -- cgit From 23291a5e1a0134aff5fe030b52d4335a6f2a18d9 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Sat, 5 Mar 2011 01:07:12 +0900 Subject: delete unnecessary DECLARE --- nova/tests/test_virt.py | 1 - 1 file changed, 1 deletion(-) (limited to 'nova') diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 5bb31659b..7ea8c0fb5 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -36,7 +36,6 @@ from nova.virt import libvirt_conn libvirt = None FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') -flags.DECLARE('compute_driver', 'nova.compute.manager') class LibvirtConnTestCase(test.TestCase): -- cgit From 68d894be2ec3b4eaa14dc5c90143f45f7db1e4b8 Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Fri, 4 Mar 2011 17:48:28 +0000 Subject: * Tests to verify correct vm-params for Windows and Linux instances --- nova/compute/api.py | 5 +- .../versions/007_add_os_type_to_instances.py | 4 +- nova/tests/db/fakes.py | 1 + nova/tests/test_xenapi.py | 99 ++++++++++++++++++---- nova/virt/xenapi/vm_utils.py | 14 +-- nova/virt/xenapi/vmops.py | 8 +- 6 files changed, 101 insertions(+), 30 deletions(-) (limited to 'nova') diff --git a/nova/compute/api.py b/nova/compute/api.py index 8bdf712a0..d79371e94 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -125,7 +125,10 @@ class API(base.Base): raise quota.QuotaError(msg, "MetadataLimitExceeded") image = self.image_service.show(context, image_id) - os_type = image['properties'].get('os_type', 'linux') + + os_type = None + if 'properties' in image and 'os_type' in image['properties']: + os_type = image['properties']['os_type'] if kernel_id is None: kernel_id = image.get('kernel_id', None) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py index 21f21b040..d6d964b95 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py @@ -34,7 +34,7 @@ instances_os_type = Column('os_type', String(length=255, convert_unicode=False, assert_unicode=None, unicode_error=None, _warn_on_bytestring=False), - nullable=False) + nullable=True) def upgrade(migrate_engine): @@ -43,5 +43,3 @@ def upgrade(migrate_engine): meta.bind = migrate_engine instances.create_column(instances_os_type) - - diff --git a/nova/tests/db/fakes.py b/nova/tests/db/fakes.py index 05bdd172e..facd6efae 100644 --- a/nova/tests/db/fakes.py +++ b/nova/tests/db/fakes.py @@ -62,6 +62,7 @@ def stub_out_db_instance_api(stubs): 'mac_address': values['mac_address'], 'vcpus': type_data['vcpus'], 'local_gb': type_data['local_gb'], + 'os_type': values['os_type'] } return FakeModel(base_options) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index b9bb6d5b4..24a5698e5 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -18,6 +18,7 @@ Test suite for XenAPI """ +import functools import stubout from nova import db @@ -41,6 +42,21 @@ from nova.tests.glance import stubs as glance_stubs FLAGS = flags.FLAGS +def stub_vm_utils_with_vdi_attached_here(function, should_return=True): + """ + vm_utils.with_vdi_attached_here needs to be stubbed out because it + calls down to the filesystem to attach a vdi. This provides a + decorator to handle that. + """ + @functools.wraps(function) + def decorated_function(self, *args, **kwargs): + orig_with_vdi_attached_here = vm_utils.with_vdi_attached_here + vm_utils.with_vdi_attached_here = lambda *x: should_return + function(self, *args, **kwargs) + vm_utils.with_vdi_attached_here = orig_with_vdi_attached_here + return decorated_function + + class XenAPIVolumeTestCase(test.TestCase): """ Unit tests for Volume operations @@ -62,6 +78,7 @@ class XenAPIVolumeTestCase(test.TestCase): 'ramdisk_id': 3, 'instance_type': 'm1.large', 'mac_address': 'aa:bb:cc:dd:ee:ff', + 'os_type': 'linux' } def _create_volume(self, size='0'): @@ -219,7 +236,7 @@ class XenAPIVMTestCase(test.TestCase): check() - def check_vm_record(self, conn): + def create_vm_record(self, conn, os_type): instances = conn.list_instances() self.assertEquals(instances, [1]) @@ -231,28 +248,63 @@ class XenAPIVMTestCase(test.TestCase): in xenapi_fake.get_all_records('VM').iteritems() if not rec['is_control_domain']] vm = vms[0] + self.vm_info = vm_info + self.vm = vm + def check_vm_record(self): # Check that m1.large above turned into the right thing. instance_type = instance_types.INSTANCE_TYPES['m1.large'] mem_kib = long(instance_type['memory_mb']) << 10 mem_bytes = str(mem_kib << 10) vcpus = instance_type['vcpus'] - self.assertEquals(vm_info['max_mem'], mem_kib) - self.assertEquals(vm_info['mem'], mem_kib) - self.assertEquals(vm['memory_static_max'], mem_bytes) - self.assertEquals(vm['memory_dynamic_max'], mem_bytes) - self.assertEquals(vm['memory_dynamic_min'], mem_bytes) - self.assertEquals(vm['VCPUs_max'], str(vcpus)) - self.assertEquals(vm['VCPUs_at_startup'], str(vcpus)) + self.assertEquals(self.vm_info['max_mem'], mem_kib) + self.assertEquals(self.vm_info['mem'], mem_kib) + self.assertEquals(self.vm['memory_static_max'], mem_bytes) + self.assertEquals(self.vm['memory_dynamic_max'], mem_bytes) + self.assertEquals(self.vm['memory_dynamic_min'], mem_bytes) + self.assertEquals(self.vm['VCPUs_max'], str(vcpus)) + self.assertEquals(self.vm['VCPUs_at_startup'], str(vcpus)) # Check that the VM is running according to Nova - self.assertEquals(vm_info['state'], power_state.RUNNING) + self.assertEquals(self.vm_info['state'], power_state.RUNNING) # Check that the VM is running according to XenAPI. - self.assertEquals(vm['power_state'], 'Running') + self.assertEquals(self.vm['power_state'], 'Running') + + def check_vm_params_for_windows(self): + self.assertEquals(self.vm['platform']['nx'], 'true') + self.assertEquals(self.vm['HVM_boot_params'], {'order': 'dc'}) + self.assertEquals(self.vm['HVM_boot_policy'], 'BIOS order') + + # check that these are not set + self.assertEquals(self.vm['PV_args'], '') + self.assertEquals(self.vm['PV_bootloader'], '') + self.assertEquals(self.vm['PV_kernel'], '') + self.assertEquals(self.vm['PV_ramdisk'], '') + + def check_vm_params_for_linux(self): + self.assertEquals(self.vm['platform']['nx'], 'false') + self.assertEquals(self.vm['PV_args'], 'clocksource=jiffies') + self.assertEquals(self.vm['PV_bootloader'], 'pygrub') + + # check that these are not set + self.assertEquals(self.vm['PV_kernel'], '') + self.assertEquals(self.vm['PV_ramdisk'], '') + self.assertEquals(self.vm['HVM_boot_params'], {}) + self.assertEquals(self.vm['HVM_boot_policy'], '') + + def check_vm_params_for_linux_with_external_kernel(self): + self.assertEquals(self.vm['platform']['nx'], 'false') + self.assertEquals(self.vm['PV_args'], 'root=/dev/xvda1') + self.assertNotEquals(self.vm['PV_kernel'], '') + self.assertNotEquals(self.vm['PV_ramdisk'], '') + + # check that these are not set + self.assertEquals(self.vm['HVM_boot_params'], {}) + self.assertEquals(self.vm['HVM_boot_policy'], '') def _test_spawn(self, image_id, kernel_id, ramdisk_id, - instance_type="m1.large"): + instance_type="m1.large", os_type="linux"): stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) values = {'name': 1, 'id': 1, @@ -263,11 +315,13 @@ class XenAPIVMTestCase(test.TestCase): 'ramdisk_id': ramdisk_id, 'instance_type': instance_type, 'mac_address': 'aa:bb:cc:dd:ee:ff', + 'os_type': os_type } conn = xenapi_conn.get_connection(False) instance = db.instance_create(values) conn.spawn(instance) - self.check_vm_record(conn) + self.create_vm_record(conn, os_type) + self.check_vm_record() def test_spawn_not_enough_memory(self): FLAGS.xenapi_image_service = 'glance' @@ -283,24 +337,37 @@ class XenAPIVMTestCase(test.TestCase): FLAGS.xenapi_image_service = 'objectstore' self._test_spawn(1, 2, 3) + @stub_vm_utils_with_vdi_attached_here def test_spawn_raw_glance(self): FLAGS.xenapi_image_service = 'glance' self._test_spawn(glance_stubs.FakeGlance.IMAGE_RAW, None, None) + self.check_vm_params_for_linux() + + def test_spawn_vhd_glance_linux(self): + FLAGS.xenapi_image_service = 'glance' + self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, + os_type="linux") + self.check_vm_params_for_linux() - def test_spawn_vhd_glance(self): + def test_spawn_vhd_glance_windows(self): FLAGS.xenapi_image_service = 'glance' - self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None) + self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, + os_type="windows") + self.check_vm_params_for_windows() def test_spawn_glance(self): FLAGS.xenapi_image_service = 'glance' self._test_spawn(glance_stubs.FakeGlance.IMAGE_MACHINE, glance_stubs.FakeGlance.IMAGE_KERNEL, glance_stubs.FakeGlance.IMAGE_RAMDISK) + self.check_vm_params_for_linux_with_external_kernel() def tearDown(self): super(XenAPIVMTestCase, self).tearDown() self.manager.delete_project(self.project) self.manager.delete_user(self.user) + self.vm_info = None + self.vm = None self.stubs.UnsetAll() def _create_instance(self): @@ -314,7 +381,8 @@ class XenAPIVMTestCase(test.TestCase): 'kernel_id': 2, 'ramdisk_id': 3, 'instance_type': 'm1.large', - 'mac_address': 'aa:bb:cc:dd:ee:ff'} + 'mac_address': 'aa:bb:cc:dd:ee:ff', + 'os_type': 'linux'} instance = db.instance_create(values) self.conn.spawn(instance) return instance @@ -360,6 +428,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): self.fake_instance = FakeInstance() self.fake_instance.id = 42 + self.fake_instance.os_type = 'linux' def assert_disk_type(self, disk_type): dt = vm_utils.VMHelper.determine_disk_image_type( diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 9c0bb5579..a26e391df 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -80,7 +80,8 @@ class VMHelper(HelperBase): """ @classmethod - def create_vm(cls, session, instance, kernel, ramdisk, use_pv_kernel=False): + def create_vm(cls, session, instance, kernel, ramdisk, + use_pv_kernel=False): """Create a VM record. Returns a Deferred that gives the new VM reference. the use_pv_kernel flag indicates whether the guest is HVM or PV @@ -319,7 +320,7 @@ class VMHelper(HelperBase): 'glance_host': FLAGS.glance_host, 'glance_port': FLAGS.glance_port, 'sr_path': get_sr_path(session), - 'os_type': instance.get('os_type', 'linux')} + 'os_type': instance.os_type} kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'upload_vhd', kwargs) @@ -524,7 +525,7 @@ class VMHelper(HelperBase): Determine whether the VM will use a paravirtualized kernel or if it will use hardware virtualization. - 1. Objectstore (any image type): + 1. Objectstore (any image type): We use plugin to figure out whether the VDI uses PV 2. Glance (VHD): then we use `os_type`, raise if not set @@ -540,7 +541,8 @@ class VMHelper(HelperBase): session, vdi_ref, disk_image_type, os_type) else: # 1. Objecstore - return cls._determine_is_pv_objectstore(session, instance_id, vdi_ref) + return cls._determine_is_pv_objectstore(session, instance_id, + vdi_ref) @classmethod def _determine_is_pv_objectstore(cls, session, instance_id, vdi_ref): @@ -564,7 +566,7 @@ class VMHelper(HelperBase): """ For a Glance image, determine if we need paravirtualization. - The relevant scenarios are: + The relevant scenarios are: 2. Glance (VHD): then we use `os_type`, raise if not set 3. Glance (DISK_RAW): use Pygrub to figure out if pv kernel is @@ -582,7 +584,7 @@ class VMHelper(HelperBase): is_pv = True elif disk_image_type == ImageType.DISK_RAW: # 3. RAW - is_pv = with_vdi_attached_here(session, vdi_ref, True, _is_vdi_pv) + is_pv = with_vdi_attached_here(session, vdi_ref, True, _is_vdi_pv) elif disk_image_type == ImageType.DISK: # 4. Disk is_pv = True diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 1edf39c5b..eedb07a50 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -87,8 +87,6 @@ class VMOps(object): vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) - os_type = instance.get('os_type', 'linux') - kernel = None if instance.kernel_id: kernel = VMHelper.fetch_image(self._session, instance.id, @@ -99,8 +97,8 @@ class VMOps(object): ramdisk = VMHelper.fetch_image(self._session, instance.id, instance.ramdisk_id, user, project, ImageType.KERNEL_RAMDISK) - use_pv_kernel = VMHelper.determine_is_pv( - self._session, instance.id, vdi_ref, disk_image_type, os_type) + use_pv_kernel = VMHelper.determine_is_pv(self._session, instance.id, + vdi_ref, disk_image_type, instance.os_type) vm_ref = VMHelper.create_vm(self._session, instance, kernel, ramdisk, use_pv_kernel) @@ -242,7 +240,7 @@ class VMOps(object): finally: self._destroy(instance, template_vm_ref, shutdown=False, destroy_kernel_ramdisk=False) - + logging.debug(_("Finished snapshot and upload for VM %s"), instance) def reboot(self, instance): -- cgit From f36b4fe22bcb187d5f426320bbe43fcf3cb1a30a Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Fri, 4 Mar 2011 14:44:29 -0500 Subject: refactor server tests to support xml and json separately --- nova/tests/api/openstack/test_servers.py | 103 ++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 35 deletions(-) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index bf934113a..9e7bc3aac 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -233,12 +233,9 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) - def _personality_dict(self, path, contents): - return {'path': path, 'contents': contents} + def _setup_mock_compute_api_for_personality(self): - def _create_instance_with_personality(self, personality): - - class FakeComputeAPI(object): + class MockComputeAPI(object): def __init__(self): self.personality_files = None @@ -255,28 +252,74 @@ class ServersTest(test.TestCase): return canned_return return stub_method - compute_api = FakeComputeAPI() + compute_api = MockComputeAPI() self.stubs.Set(nova.compute, 'API', make_stub_method(compute_api)) self.stubs.Set(nova.api.openstack.servers.Controller, '_get_kernel_ramdisk_from_image', make_stub_method((1, 1))) self.stubs.Set(nova.api.openstack.common, 'get_image_id_from_image_hash', make_stub_method(2)) - body = dict(server=dict( - name='server_test', imageId=2, flavorId=2, - metadata={}, - personality=personality)) - if personality is None: - del body['server']['personality'] - + return compute_api + + def _create_personality_request_dict(self, personality_files): + server = {} + server['name'] = 'new-server-test' + server['imageId'] = 1 + server['flavorId'] = 1 + if personality_files is not None: + personalities = [] + for path, contents in personality_files: + personalities.append({'path': path, 'contents': contents}) + server['personality'] = personalities + return {'server': server} + + def _create_personality_request_json(self, personality_files): + body_dict = self._create_personality_request_dict(personality_files) req = webob.Request.blank('/v1.0/servers') + req.content_type = 'application/json' req.method = 'POST' - req.body = json.dumps(body) - return (req, req.get_response(fakes.wsgi_app()), - compute_api.personality_files) + req.body = json.dumps(body_dict) + return req + + def _format_xml_request_body(self, body_dict): + server = body_dict['server'] + body_parts = [] + body_parts.extend([ + '', + '' % ( + server['name'], server['imageId'], server['flavorId'])]) + if 'metadata' in server: + metadata = server['metadata'] + body_parts.append('') + for item in metadata.iteritems(): + body_parts.append('%s' % item) + body_parts.append('') + if 'personality' in server: + personalities = server['personality'] + body_parts.append('') + for item in personalities.iteritems(): + body_parts.append('%s' % item) + body_parts.append('') + body_parts.append('') + return ''.join(body_parts) + + def _create_personality_request_xml(self, personality_files): + body_dict = self._create_personality_request_dict(personality_files) + req = webob.Request.blank('/v1.0/servers') + req.content_type = 'application/xml' + req.method = 'POST' + req.body = self._format_xml_request_body(body_dict) + return req + + def _create_instance_with_personality_json(self, personality): + compute_api = self._setup_mock_compute_api_for_personality() + request = self._create_personality_request_json(personality) + response = request.get_response(fakes.wsgi_app()) + return (request, response, compute_api.personality_files) def test_create_instance_with_no_personality(self): request, response, personality_files = \ - self._create_instance_with_personality(personality=None) + self._create_instance_with_personality_json(personality=None) self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, []) @@ -284,18 +327,18 @@ class ServersTest(test.TestCase): path = '/my/file/path' contents = '#!/bin/bash\necho "Hello, World!"\n' b64contents = base64.b64encode(contents) - personality = [self._personality_dict(path, b64contents)] + personality = [(path, b64contents)] request, response, personality_files = \ - self._create_instance_with_personality(personality) + self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, [(path, contents)]) def test_create_instance_with_personality_with_non_b64_content(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Oh no!"\n' - personality = [self._personality_dict(path, contents)] + personality = [(path, contents)] request, response, personality_files = \ - self._create_instance_with_personality(personality) + self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 400) self.assertEquals(personality_files, None) @@ -307,31 +350,21 @@ class ServersTest(test.TestCase): ] personality = [] for path, content in files: - personality.append(self._personality_dict( - path, base64.b64encode(content))) + personality.append((path, base64.b64encode(content))) request, response, personality_files = \ - self._create_instance_with_personality(personality) + self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, files) def test_create_instance_personality_empty_content(self): path = '/my/file/path' contents = '' - personality = [self._personality_dict(path, contents)] + personality = [(path, contents)] request, response, personality_files = \ - self._create_instance_with_personality(personality) + self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, [(path, contents)]) - def test_create_instance_personality_not_a_list(self): - path = '/my/file/path' - contents = 'myfilecontents' - personality = self._personality_dict(path, contents) - request, response, personality_files = \ - self._create_instance_with_personality(personality) - self.assertEquals(response.status_int, 400) - self.assertEquals(personality_files, None) - def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' -- cgit From a38e6c67c37a4d3336cf1dc3717fd5612a474183 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Fri, 4 Mar 2011 14:45:31 -0500 Subject: remove xml testing infrastructure since it is not feasible to use at present --- nova/tests/api/openstack/test_servers.py | 31 ------------------------------- 1 file changed, 31 deletions(-) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 9e7bc3aac..c125e6192 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -280,37 +280,6 @@ class ServersTest(test.TestCase): req.body = json.dumps(body_dict) return req - def _format_xml_request_body(self, body_dict): - server = body_dict['server'] - body_parts = [] - body_parts.extend([ - '', - '' % ( - server['name'], server['imageId'], server['flavorId'])]) - if 'metadata' in server: - metadata = server['metadata'] - body_parts.append('') - for item in metadata.iteritems(): - body_parts.append('%s' % item) - body_parts.append('') - if 'personality' in server: - personalities = server['personality'] - body_parts.append('') - for item in personalities.iteritems(): - body_parts.append('%s' % item) - body_parts.append('') - body_parts.append('') - return ''.join(body_parts) - - def _create_personality_request_xml(self, personality_files): - body_dict = self._create_personality_request_dict(personality_files) - req = webob.Request.blank('/v1.0/servers') - req.content_type = 'application/xml' - req.method = 'POST' - req.body = self._format_xml_request_body(body_dict) - return req - def _create_instance_with_personality_json(self, personality): compute_api = self._setup_mock_compute_api_for_personality() request = self._create_personality_request_json(personality) -- cgit From f3c1c99ca0f6f3164430b33f46772ef8bdc87b70 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 4 Mar 2011 19:54:19 +0000 Subject: move the id wrapping into cloud layer instead of image_service --- nova/api/ec2/cloud.py | 33 ++++++++++++++++++++++++--------- nova/image/s3.py | 28 +++++----------------------- 2 files changed, 29 insertions(+), 32 deletions(-) (limited to 'nova') diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index aa1dcbe33..3ea3fa07e 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -843,10 +843,19 @@ class CloudController(object): self.compute_api.update(context, instance_id=instance_id, **kwargs) return True + _type_prefix_map = {'machine': 'ami', + 'kernel': 'aki', + 'ramdisk': 'ari'} + + def _image_ec2_id(self, image_id, image_type): + prefix = self._type_prefix_map[image_type] + template = prefix + '-%08x' + return ec2utils.id_to_ec2_id(int(image_id), template=template) + def _format_image(self, image): """Convert from format defined by BaseImageService to S3 format.""" i = {} - i['imageId'] = image.get('id') + i['imageId'] = self._image_ec2_id(image.get('id'), image.get('type')) i['kernelId'] = image['properties'].get('kernel_id') i['ramdiskId'] = image['properties'].get('ramdisk_id') i['imageOwnerId'] = image['properties'].get('owner_id') @@ -863,7 +872,8 @@ class CloudController(object): images = [] for ec2_id in image_id: try: - image = self.image_service.show(context, ec2_id) + internal_id = ec2utils.ec2_id_to_id(ec2_id) + image = self.image_service.show(context, internal_id) except exception.NotFound: raise exception.NotFound(_('Image %s not found') % ec2_id) @@ -875,14 +885,16 @@ class CloudController(object): def deregister_image(self, context, image_id, **kwargs): LOG.audit(_("De-registering image %s"), image_id, context=context) - self.image_service.delete(context, image_id) + internal_id = ec2utils.ec2_id_to_id(image_id) + self.image_service.delete(context, internal_id) return {'imageId': image_id} def register_image(self, context, image_location=None, **kwargs): if image_location is None and 'name' in kwargs: image_location = kwargs['name'] - image = {"image_location": image_location} - image_id = self.image_service.create(context, image) + metadata = {"image_location": image_location} + image = self.image_service.create(context, metadata) + image_id = self._image_ec2_id(image['id'], image['type']) msg = _("Registered image %(image_location)s with" " id %(image_id)s") % locals() LOG.audit(msg, context=context) @@ -893,8 +905,9 @@ class CloudController(object): raise exception.ApiError(_('attribute not supported: %s') % attribute) try: + internal_id = ec2utils.ec2_id_to_id(image_id) image = self._format_image(self.image_service.show(context, - image_id)) + internal_id)) except (IndexError, exception.NotFound): raise exception.NotFound(_('Image %s not found') % image_id) result = {'image_id': image_id, 'launchPermission': []} @@ -917,13 +930,15 @@ class CloudController(object): LOG.audit(_("Updating image %s publicity"), image_id, context=context) try: - metadata = self.image_service.show(context, image_id) + internal_id = ec2utils.ec2_id_to_id(image_id) + metadata = self.image_service.show(context, internal_id) except exception.NotFound: raise exception.NotFound(_('Image %s not found') % image_id) del(metadata['id']) metadata['properties']['is_public'] = (operation_type == 'add') - return self.image_service.update(context, image_id, metadata) + return self.image_service.update(context, internal_id, metadata) def update_image(self, context, image_id, **kwargs): - result = self.image_service.update(context, image_id, dict(kwargs)) + internal_id = ec2utils.ec2_id_to_id(image_id) + result = self.image_service.update(context, internal_id, dict(kwargs)) return result diff --git a/nova/image/s3.py b/nova/image/s3.py index e9542c7bd..c7446f4b0 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -36,7 +36,6 @@ from nova import flags from nova import utils from nova.auth import manager from nova.image import service -from nova.api.ec2 import ec2utils FLAGS = flags.FLAGS @@ -44,17 +43,6 @@ flags.DEFINE_string('image_decryption_dir', '/tmp', 'parent dir for tempdir used for image decryption') -_type_prefix_map = {'machine': 'ami', - 'kernel': 'aki', - 'ramdisk': 'ari'} - - -def image_ec2_id(image_id, image_type): - prefix = _type_prefix_map[image_type] - template = prefix + '-%08x' - return ec2utils.id_to_ec2_id(int(image_id), template=template) - - class S3ImageService(service.BaseImageService): def __init__(self, service=None, *args, **kwargs): if service == None: @@ -62,23 +50,20 @@ class S3ImageService(service.BaseImageService): self.service = service self.service.__init__(*args, **kwargs) - def create(self, context, properties, data=None): - """image should contain image_location""" - image_id, metadata = self._s3_create(context, properties) - return image_ec2_id(image_id, metadata['type']) + def create(self, context, metadata, data=None): + """metadata should contain image_location""" + image = self._s3_create(context, metadata) + return image def delete(self, context, image_id): # FIXME(vish): call to show is to check filter self.show(context, image_id) - image_id = ec2utils.ec2_id_to_id(image_id) self.service.delete(context, image_id) def update(self, context, image_id, metadata, data=None): # FIXME(vish): call to show is to check filter self.show(context, image_id) - image_id = ec2utils.ec2_id_to_id(image_id) image = self.service.update(context, image_id, metadata, data) - image['id'] = image_ec2_id(image['id'], image['type']) return image def index(self, context): @@ -103,16 +88,13 @@ class S3ImageService(service.BaseImageService): for image in images: if not S3ImageService._is_visible(context, image): continue - image['id'] = image_ec2_id(image['id'], image['type']) filtered.append(image) return filtered def show(self, context, image_id): - image_id = ec2utils.ec2_id_to_id(image_id) image = self.service.show(context, image_id) if not self._is_visible(context, image): raise exception.NotFound - image['id'] = image_ec2_id(image['id'], image['type']) return image @staticmethod @@ -233,7 +215,7 @@ class S3ImageService(service.BaseImageService): eventlet.spawn_n(delayed_create) - return image_id, metadata + return image @staticmethod def _decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, -- cgit From 10668b87f46a1fb5d039f6e7d7a7a55b89d7602a Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Fri, 4 Mar 2011 17:04:41 -0500 Subject: respond well if personality attribute is incomplete --- nova/api/openstack/servers.py | 3 ++ nova/tests/api/openstack/test_servers.py | 48 ++++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 5 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 8b7b20b92..7c620dbc6 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -153,6 +153,9 @@ class Controller(wsgi.Controller): try: path = item['path'] contents = item['contents'] + except KeyError, key: + expl = 'Bad personality format: missing %s' % key + raise exc.HTTPBadRequest(explanation=expl) except TypeError: raise exc.HTTPBadRequest(explanation='Bad personality format') try: diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index c125e6192..8fb5a9aec 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -272,19 +272,24 @@ class ServersTest(test.TestCase): server['personality'] = personalities return {'server': server} - def _create_personality_request_json(self, personality_files): - body_dict = self._create_personality_request_dict(personality_files) + def _get_create_request_json(self, body_dict): req = webob.Request.blank('/v1.0/servers') req.content_type = 'application/json' req.method = 'POST' req.body = json.dumps(body_dict) return req - def _create_instance_with_personality_json(self, personality): + def _run_create_instance_with_mock_compute_api(self, request): compute_api = self._setup_mock_compute_api_for_personality() - request = self._create_personality_request_json(personality) response = request.get_response(fakes.wsgi_app()) - return (request, response, compute_api.personality_files) + return compute_api, response + + def _create_instance_with_personality_json(self, personality): + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + return request, response, compute_api.personality_files def test_create_instance_with_no_personality(self): request, response, personality_files = \ @@ -302,6 +307,39 @@ class ServersTest(test.TestCase): self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, [(path, contents)]) + def test_create_instance_with_personality_no_path(self): + personality = [('/remove/this/path', + base64.b64encode('my\n\file\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + del body_dict['server']['personality'][0]['path'] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + + def test_create_instance_with_personality_no_contents(self): + personality = [('/test/path', + base64.b64encode('remove\nthese\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + del body_dict['server']['personality'][0]['contents'] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + + def test_create_instance_with_personality_not_a_list(self): + personality = [('/test/path', base64.b64encode('test\ncontents\n'))] + body_dict = self._create_personality_request_dict(personality) + body_dict['server']['personality'] = \ + body_dict['server']['personality'][0] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + def test_create_instance_with_personality_with_non_b64_content(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Oh no!"\n' -- cgit From e63cd9d5dc856f81477cf6c0e6c77ed7d1f4d70c Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Fri, 4 Mar 2011 22:17:53 +0000 Subject: * os_type is no longer `not null` --- .../versions/007_add_os_type_to_instances.py | 45 ----------------- .../versions/009_add_os_type_to_instances.py | 56 ++++++++++++++++++++++ nova/virt/xenapi/vm_utils.py | 8 ++-- 3 files changed, 60 insertions(+), 49 deletions(-) delete mode 100644 nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py (limited to 'nova') diff --git a/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py deleted file mode 100644 index d6d964b95..000000000 --- a/nova/db/sqlalchemy/migrate_repo/versions/007_add_os_type_to_instances.py +++ /dev/null @@ -1,45 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import * -from migrate import * - -from nova import log as logging - - -meta = MetaData() - -instances = Table('instances', meta, - Column('id', Integer(), primary_key=True, nullable=False), - ) - -# FIXME(dubs) should this be not null? Maybe create as nullable, then -# populate all existing rows with 'linux', then adding not null constraint. -instances_os_type = Column('os_type', - String(length=255, convert_unicode=False, - assert_unicode=None, unicode_error=None, - _warn_on_bytestring=False), - nullable=True) - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; - # bind migrate_engine to your metadata - meta.bind = migrate_engine - - instances.create_column(instances_os_type) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py new file mode 100644 index 000000000..a50f31e16 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py @@ -0,0 +1,56 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from sqlalchemy.sql import text +from migrate import * + +from nova import log as logging + + +meta = MetaData() + +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +# FIXME(dubs) should this be not null? Maybe create as nullable, then +# populate all existing rows with 'linux', then adding not null constraint. +instances_os_type = Column('os_type', + String(length=255, convert_unicode=False, + assert_unicode=None, unicode_error=None, + _warn_on_bytestring=False), + nullable=True) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + instances.create_column(instances_os_type) + migrate_engine.execute(instances.update()\ + .where(instances.c.os_type==None)\ + .values(os_type='linux')) + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + + instances.drop_column('os_type') + diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 7bff81b66..150824400 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -117,7 +117,7 @@ class VMHelper(HelperBase): 'memory_target': mem, 'name_description': '', 'name_label': instance.name, -# 'other_config': {'allowvssprovider': False}, + 'other_config': {'allowvssprovider': False}, 'other_config': {}, 'PCI_bus': '', 'platform': {'acpi': 'true', 'apic': 'true', 'pae': 'true', @@ -313,14 +313,14 @@ class VMHelper(HelperBase): logging.debug(_("Asking xapi to upload %(vdi_uuids)s as" " ID %(image_id)s") % locals()) - # TODO(dubs): os_type is currently defaulting to linux, we actually - # want to make this a NOT NULL column and require it to be specified. + os_type = instance.os_type and instance.os_type or 'linux' + params = {'vdi_uuids': vdi_uuids, 'image_id': image_id, 'glance_host': FLAGS.glance_host, 'glance_port': FLAGS.glance_port, 'sr_path': get_sr_path(session), - 'os_type': instance.os_type} + 'os_type': os_type} kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'upload_vhd', kwargs) -- cgit From 7af17cbef6f3e1c5b052133e40e0edbd8ca9ffb3 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Sun, 6 Mar 2011 10:41:24 -0500 Subject: select cleanups --- nova/api/openstack/servers.py | 2 +- nova/quota.py | 14 ++++++-------- nova/virt/xenapi/vmops.py | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 7c620dbc6..93f504f91 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -153,7 +153,7 @@ class Controller(wsgi.Controller): try: path = item['path'] contents = item['contents'] - except KeyError, key: + except KeyError as key: expl = 'Bad personality format: missing %s' % key raise exc.HTTPBadRequest(explanation=expl) except TypeError: diff --git a/nova/quota.py b/nova/quota.py index 4b777624c..1d79f9ef2 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -46,14 +46,12 @@ flags.DEFINE_integer('quota_personality_max_path_bytes', 255, def get_quota(context, project_id): - rval = { - 'instances': FLAGS.quota_instances, - 'cores': FLAGS.quota_cores, - 'volumes': FLAGS.quota_volumes, - 'gigabytes': FLAGS.quota_gigabytes, - 'floating_ips': FLAGS.quota_floating_ips, - 'metadata_items': FLAGS.quota_metadata_items, - } + rval = {'instances': FLAGS.quota_instances, + 'cores': FLAGS.quota_cores, + 'volumes': FLAGS.quota_volumes, + 'gigabytes': FLAGS.quota_gigabytes, + 'floating_ips': FLAGS.quota_floating_ips, + 'metadata_items': FLAGS.quota_metadata_items} try: quota = db.quota_get(context, project_id) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 89d58a664..cf4bedaa9 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -316,7 +316,7 @@ class VMOps(object): def inject_file(self, instance, path, contents): """Write a file to the VM instance. The path to which it is to be - written and the contents of the file need to be supplied; both should + written and the contents of the file need to be supplied; both will be base64-encoded to prevent errors with non-ASCII characters being transmitted. If the agent does not support file injection, or the user has disabled it, a NotImplementedError will be raised. -- cgit From aa4b8a557505108341d603659a5456d10d5f9632 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Sun, 6 Mar 2011 12:06:39 -0500 Subject: avoid possible string/int comparison problems --- nova/quota.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/quota.py b/nova/quota.py index 1d79f9ef2..fdef42bc5 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -115,17 +115,17 @@ def allowed_metadata_items(context, num_metadata_items): def allowed_personality_files(context): """Return the number of personality files allowed""" - return FLAGS.quota_personality_max_files + return int(FLAGS.quota_personality_max_files) def allowed_personality_content_bytes(context): """Return the number of bytes allowed per personality content""" - return FLAGS.quota_personality_max_content_bytes + return int(FLAGS.quota_personality_max_content_bytes) def allowed_personality_path_bytes(context): """Return the number of bytes allowed in a personality file path""" - return FLAGS.quota_personality_max_path_bytes + return int(FLAGS.quota_personality_max_path_bytes) class QuotaError(exception.ApiError): -- cgit From a5bee00af4d6ec3eed6ed0abd866948f4510f041 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 7 Mar 2011 01:25:01 +0000 Subject: make compute get the new images properly, fix a bunch of tests, and provide conversion commands --- nova/api/ec2/cloud.py | 74 ++++++++++++++++++++++++++++---------------- nova/api/ec2/ec2utils.py | 6 +++- nova/compute/api.py | 4 +-- nova/image/glance.py | 35 ++++++++++++++++++--- nova/image/local.py | 52 ++++++++++++++++++++++++------- nova/image/s3.py | 33 +++++++++++++------- nova/image/service.py | 18 ++++++++--- nova/tests/test_cloud.py | 16 +++++----- nova/tests/test_compute.py | 12 +++---- nova/tests/test_console.py | 2 +- nova/tests/test_quota.py | 32 +++++++++++-------- nova/tests/test_scheduler.py | 4 +-- nova/tests/test_volume.py | 2 +- nova/virt/images.py | 25 +++++++++------ nova/virt/libvirt_conn.py | 8 +++-- 15 files changed, 216 insertions(+), 107 deletions(-) (limited to 'nova') diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 3ea3fa07e..496e944fe 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -146,10 +146,13 @@ class CloudController(object): floating_ip = db.instance_get_floating_address(ctxt, instance_ref['id']) ec2_id = ec2utils.id_to_ec2_id(instance_ref['id']) + image_ec2_id = self._image_ec2_id(instance_ref['image_id'], 'machine') + k_ec2_id = self._image_ec2_id(instance_ref['kernel_id'], 'kernel') + r_ec2_id = self._image_ec2_id(instance_ref['ramdisk_id'], 'ramdisk') data = { 'user-data': base64.b64decode(instance_ref['user_data']), 'meta-data': { - 'ami-id': instance_ref['image_id'], + 'ami-id': image_ec2_id, 'ami-launch-index': instance_ref['launch_index'], 'ami-manifest-path': 'FIXME', 'block-device-mapping': { @@ -164,12 +167,12 @@ class CloudController(object): 'instance-type': instance_ref['instance_type'], 'local-hostname': hostname, 'local-ipv4': address, - 'kernel-id': instance_ref['kernel_id'], + 'kernel-id': k_ec2_id, + 'ramdisk-id': r_ec2_id, 'placement': {'availability-zone': availability_zone}, 'public-hostname': hostname, 'public-ipv4': floating_ip or '', 'public-keys': keys, - 'ramdisk-id': instance_ref['ramdisk_id'], 'reservation-id': instance_ref['reservation_id'], 'security-groups': '', 'mpi': mpi}} @@ -679,7 +682,7 @@ class CloudController(object): instance_id = instance['id'] ec2_id = ec2utils.id_to_ec2_id(instance_id) i['instanceId'] = ec2_id - i['imageId'] = instance['image_id'] + i['imageId'] = self._image_ec2_id(instance['image_id']) i['instanceState'] = { 'code': instance['state'], 'name': instance['state_description']} @@ -782,13 +785,15 @@ class CloudController(object): def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) if kwargs.get('kernel_id'): - kwargs['kernel_id'] = ec2utils.ec2_id_to_id(kwargs['kernel_id']) + kernel = self._get_image(context, kwargs['kernel_id']) + kwargs['kernel_id'] = kernel['id'] if kwargs.get('ramdisk_id'): - kwargs['ramdisk_id'] = ec2utils.ec2_id_to_id(kwargs['ramdisk_id']) + ramdisk = self._get_image(context, kwargs['ramdisk_id']) + kwargs['ramdisk_id'] = ramdisk['id'] instances = self.compute_api.create(context, instance_type=instance_types.get_by_type( kwargs.get('instance_type', None)), - image_id=ec2utils.ec2_id_to_id(kwargs['image_id']), + image_id=self._get_image(context, kwargs['image_id'])['id'], min_count=int(kwargs.get('min_count', max_count)), max_count=max_count, kernel_id=kwargs.get('kernel_id'), @@ -847,17 +852,34 @@ class CloudController(object): 'kernel': 'aki', 'ramdisk': 'ari'} - def _image_ec2_id(self, image_id, image_type): + def _image_ec2_id(self, image_id, image_type='machine'): prefix = self._type_prefix_map[image_type] template = prefix + '-%08x' return ec2utils.id_to_ec2_id(int(image_id), template=template) + def _get_image(self, context, ec2_id): + try: + internal_id = ec2utils.ec2_id_to_id(ec2_id) + return self.image_service.show(context, internal_id) + except exception.NotFound: + return self.image_service.show_by_name(context, ec2_id) + + def _format_image(self, image): """Convert from format defined by BaseImageService to S3 format.""" i = {} - i['imageId'] = self._image_ec2_id(image.get('id'), image.get('type')) - i['kernelId'] = image['properties'].get('kernel_id') - i['ramdiskId'] = image['properties'].get('ramdisk_id') + ec2_id = self._image_ec2_id(image.get('id'), image.get('type')) + name = image.get('name') + if name: + i['imageId'] = "%s (%s)" % (ec2_id, name) + else: + i['imageId'] = ec2_id + kernel_id = image['properties'].get('kernel_id') + if kernel_id: + i['kernelId'] = self._image_ec2_id(kernel_id, 'kernel') + ramdisk_id = image['properties'].get('ramdisk_id') + if ramdisk_id: + i['ramdiskId'] = self._image_ec2_id(ramdisk_id, 'ramdisk') i['imageOwnerId'] = image['properties'].get('owner_id') i['imageLocation'] = image['properties'].get('image_location') i['imageState'] = image['properties'].get('image_state') @@ -872,8 +894,7 @@ class CloudController(object): images = [] for ec2_id in image_id: try: - internal_id = ec2utils.ec2_id_to_id(ec2_id) - image = self.image_service.show(context, internal_id) + image = self._get_image(context, ec2_id) except exception.NotFound: raise exception.NotFound(_('Image %s not found') % ec2_id) @@ -885,16 +906,17 @@ class CloudController(object): def deregister_image(self, context, image_id, **kwargs): LOG.audit(_("De-registering image %s"), image_id, context=context) - internal_id = ec2utils.ec2_id_to_id(image_id) + image = self._get_image(context, image_id) + internal_id = image['id'] self.image_service.delete(context, internal_id) return {'imageId': image_id} def register_image(self, context, image_location=None, **kwargs): if image_location is None and 'name' in kwargs: image_location = kwargs['name'] - metadata = {"image_location": image_location} + metadata = {'properties': {'image_location': image_location}} image = self.image_service.create(context, metadata) - image_id = self._image_ec2_id(image['id'], image['type']) + image_id = self._image_ec2_id(image['id'], image['type']) msg = _("Registered image %(image_location)s with" " id %(image_id)s") % locals() LOG.audit(msg, context=context) @@ -905,13 +927,11 @@ class CloudController(object): raise exception.ApiError(_('attribute not supported: %s') % attribute) try: - internal_id = ec2utils.ec2_id_to_id(image_id) - image = self._format_image(self.image_service.show(context, - internal_id)) - except (IndexError, exception.NotFound): + image = self._get_image(context, image_id) + except exception.NotFound: raise exception.NotFound(_('Image %s not found') % image_id) - result = {'image_id': image_id, 'launchPermission': []} - if image['isPublic']: + result = {'imageId': image_id, 'launchPermission': []} + if image['properties']['is_public']: result['launchPermission'].append({'group': 'all'}) return result @@ -930,13 +950,13 @@ class CloudController(object): LOG.audit(_("Updating image %s publicity"), image_id, context=context) try: - internal_id = ec2utils.ec2_id_to_id(image_id) - metadata = self.image_service.show(context, internal_id) + image = self._get_image(context, image_id) except exception.NotFound: raise exception.NotFound(_('Image %s not found') % image_id) - del(metadata['id']) - metadata['properties']['is_public'] = (operation_type == 'add') - return self.image_service.update(context, internal_id, metadata) + internal_id = image['id'] + del(image['id']) + image['properties']['is_public'] = (operation_type == 'add') + return self.image_service.update(context, internal_id, image) def update_image(self, context, image_id, **kwargs): internal_id = ec2utils.ec2_id_to_id(image_id) diff --git a/nova/api/ec2/ec2utils.py b/nova/api/ec2/ec2utils.py index 0ea22c0e6..e4df80cf8 100644 --- a/nova/api/ec2/ec2utils.py +++ b/nova/api/ec2/ec2utils.py @@ -16,10 +16,14 @@ # License for the specific language governing permissions and limitations # under the License. +from nova import exception def ec2_id_to_id(ec2_id): """Convert an ec2 ID (i-[base 16 number]) to an instance id (int)""" - return int(ec2_id.split('-')[-1], 16) + try: + return int(ec2_id.split('-')[-1], 16) + except ValueError: + raise exception.NotFound(_("Id %s Not Found") % ec2_id) def id_to_ec2_id(instance_id, template='i-%08x'): diff --git a/nova/compute/api.py b/nova/compute/api.py index 35a7d7bc0..58118121a 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -126,9 +126,9 @@ class API(base.Base): image = self.image_service.show(context, image_id) if kernel_id is None: - kernel_id = image.get('kernel_id', None) + kernel_id = image['properties'].get('kernel_id', None) if ramdisk_id is None: - ramdisk_id = image.get('ramdisk_id', None) + ramdisk_id = image['properties'].get('ramdisk_id', None) # FIXME(sirp): is there a way we can remove null_kernel? # No kernel and ramdisk for raw images if kernel_id == str(FLAGS.null_kernel): diff --git a/nova/image/glance.py b/nova/image/glance.py index 7db94c0d4..fb383f5e6 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -17,9 +17,6 @@ """Implementation of an image service that uses Glance as the backend""" from __future__ import absolute_import -import httplib -import json -import urlparse from glance.common import exception as glance_exception @@ -55,16 +52,44 @@ class GlanceImageService(service.BaseImageService): """ return self.client.get_images_detailed() - def show(self, context, id): + def show(self, context, image_id): """ Returns a dict containing image data for the given opaque image id. """ try: - image = self.client.get_image_meta(id) + image = self.client.get_image_meta(image_id) except glance_exception.NotFound: raise exception.NotFound return image + def show_by_name(self, context, name): + """ + Returns a dict containing image data for the given name. + """ + # TODO(vish): replace this with more efficient call when glance + # supports it. + images = self.detail(context) + image = None + for cantidate in images: + if name == cantidate.get('name'): + image = cantidate + break + if image == None: + raise exception.NotFound + return image + + def get(self, context, image_id, data): + """ + Calls out to Glance for metadata and data and writes data. + """ + try: + metadata, image_chunks = self.client.get_image(image_id) + except glance_exception.NotFound: + raise exception.NotFound + for chunk in image_chunks: + data.write(chunk) + return metadata + def create(self, context, metadata, data=None): """ Store the image data and return the new image id. diff --git a/nova/image/local.py b/nova/image/local.py index 6fa648b6b..c4ac3baaa 100644 --- a/nova/image/local.py +++ b/nova/image/local.py @@ -42,13 +42,12 @@ class LocalImageService(service.BaseImageService): def _path_to(self, image_id, fname='info.json'): if fname: - return os.path.join(self._path, str(image_id), fname) - return os.path.join(self._path, str(image_id)) + return os.path.join(self._path, '%08x' % int(image_id), fname) + return os.path.join(self._path, '%08x' % int(image_id)) def _ids(self): """The list of all image ids.""" - return [int(i) for i in os.listdir(self._path) - if unicode(i).isnumeric()] + return [int(i, 16) for i in os.listdir(self._path)] def index(self, context): return [dict(image_id=i['id'], name=i.get('name')) @@ -68,27 +67,56 @@ class LocalImageService(service.BaseImageService): try: with open(self._path_to(image_id)) as metadata_file: return json.load(metadata_file) - except IOError: + except (IOError, ValueError): raise exception.NotFound + def show_by_name(self, context, name): + """Returns a dict containing image data for the given name.""" + # NOTE(vish): Not very efficient, but the local image service + # is for testing so it should be fine. + images = self.detail(context) + image = None + for cantidate in images: + if name == cantidate.get('name'): + image = cantidate + break + if image == None: + raise exception.NotFound + return image + + def get(self, context, image_id, data): + """Get image and metadata.""" + try: + with open(self._path_to(image_id)) as metadata_file: + metadata = json.load(metadata_file) + with open(self._path_to(image_id, 'image')) as image_file: + shutil.copyfileobj(image_file, data) + except (IOError, ValueError): + raise exception.NotFound + return metadata + def create(self, context, metadata, data=None): - """Store the image data and return the new image id.""" + """Store the image data and return the new image.""" image_id = random.randint(0, 2 ** 31 - 1) image_path = self._path_to(image_id, None) if not os.path.exists(image_path): os.mkdir(image_path) - return self.update(context, image_id, metadata, data) + return self.update(context, image_id, metadata, data) def update(self, context, image_id, metadata, data=None): """Replace the contents of the given image with the new data.""" metadata['id'] = image_id try: - with open(self._path_to(image_id), 'w') as metadata_file: - json.dump(metadata, metadata_file) if data: - with open(self._path_to(image_id, 'image'), 'w') as image_file: + location = self._path_to(image_id, 'image') + with open(location, 'w') as image_file: shutil.copyfileobj(data, image_file) - except IOError: + # NOTE(vish): update metadata similarly to glance + metadata['status'] = 'active' + metadata['location'] = location + with open(self._path_to(image_id), 'w') as metadata_file: + json.dump(metadata, metadata_file) + except (IOError, ValueError): raise exception.NotFound return metadata @@ -99,7 +127,7 @@ class LocalImageService(service.BaseImageService): """ try: shutil.rmtree(self._path_to(image_id, None)) - except IOError: + except (IOError, ValueError): raise exception.NotFound def delete_all(self): diff --git a/nova/image/s3.py b/nova/image/s3.py index c7446f4b0..ab6eea8cf 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -36,6 +36,7 @@ from nova import flags from nova import utils from nova.auth import manager from nova.image import service +from nova.api.ec2 import ec2utils FLAGS = flags.FLAGS @@ -51,7 +52,7 @@ class S3ImageService(service.BaseImageService): self.service.__init__(*args, **kwargs) def create(self, context, metadata, data=None): - """metadata should contain image_location""" + """metadata['properties'] should contain image_location""" image = self._s3_create(context, metadata) return image @@ -97,6 +98,12 @@ class S3ImageService(service.BaseImageService): raise exception.NotFound return image + def show_by_name(self, context, name): + image = self.service.show_by_name(context, name) + if not self._is_visible(context, image): + raise exception.NotFound + return image + @staticmethod def _conn(context): # TODO(vish): is there a better way to get creds to sign @@ -119,10 +126,12 @@ class S3ImageService(service.BaseImageService): key.get_contents_to_filename(local_filename) return local_filename - def _s3_create(self, context, properties): + def _s3_create(self, context, metadata): + """Gets a manifext from s3 and makes an image""" + image_path = tempfile.mkdtemp(dir=FLAGS.image_decryption_dir) - image_location = properties['image_location'] + image_location = metadata['properties']['image_location'] bucket_name = image_location.split("/")[0] manifest_path = image_location[len(bucket_name) + 1:] bucket = self._conn(context).get_bucket(bucket_name) @@ -153,25 +162,27 @@ class S3ImageService(service.BaseImageService): except: arch = 'x86_64' - properties.update({'owner_id': context.project_id, - 'architecture': arch}) + properties = metadata['properties'] + properties['owner_id'] = context.project_id + properties['architecture'] = arch if kernel_id: - properties['kernel_id'] = kernel_id + properties['kernel_id'] = ec2utils.ec2_id_to_id(kernel_id) if ramdisk_id: - properties['ramdisk_id'] = ramdisk_id + properties['ramdisk_id'] = ec2utils.ec2_id_to_id(ramdisk_id) properties['is_public'] = False - metadata = {'type': image_type, - 'status': 'queued', - 'is_public': True, - 'properties': properties} + metadata.update({'type': image_type, + 'status': 'queued', + 'is_public': True, + 'properties': properties}) metadata['properties']['image_state'] = 'pending' image = self.service.create(context, metadata) image_id = image['id'] def delayed_create(): + """This handles the fetching and decrypting of the part files.""" parts = [] for fn_element in manifest.find("image").getiterator("filename"): part = self._download_file(bucket, fn_element.text, image_path) diff --git a/nova/image/service.py b/nova/image/service.py index e429955f4..c09052cab 100644 --- a/nova/image/service.py +++ b/nova/image/service.py @@ -56,9 +56,9 @@ class BaseImageService(object): """ raise NotImplementedError - def show(self, context, id): + def show(self, context, image_id): """ - Returns a dict containing image data for the given opaque image id. + Returns a dict containing image metadata for the given opaque image id. :retval a mapping with the following signature: @@ -76,9 +76,19 @@ class BaseImageService(object): """ raise NotImplementedError + def get(self, context, data): + """ + Returns a dict containing image metadata and writes image data to data. + + :param data: a file-like object to hold binary image data + + :raises NotFound if the image does not exist + """ + raise NotImplementedError + def create(self, context, metadata, data=None): """ - Store the image data and return the new image id. + Store the image metadata and data and return the new image id. :raises AlreadyExists if the image already exist. @@ -86,7 +96,7 @@ class BaseImageService(object): raise NotImplementedError def update(self, context, image_id, metadata, data=None): - """Replace the contents of the given image with the new data. + """Update the given image with the new metadata and data. :raises NotFound if the image does not exist. diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 7d7b91658..8d2cd9573 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -78,10 +78,11 @@ class CloudTestCase(test.TestCase): project=self.project) host = self.network.get_network_host(self.context.elevated()) - def fake_image_show(meh, context, id): - return dict(kernelId=1, ramdiskId=1) + def fake_show(meh, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}} - self.stubs.Set(local.LocalImageService, 'show', fake_image_show) + self.stubs.Set(local.LocalImageService, 'show', fake_show) + self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show) def tearDown(self): network_ref = db.project_get_network(self.context, @@ -195,8 +196,10 @@ class CloudTestCase(test.TestCase): def test_describe_instances(self): """Makes sure describe_instances works and filters results.""" inst1 = db.instance_create(self.context, {'reservation_id': 'a', + 'image_id': 1, 'host': 'host1'}) inst2 = db.instance_create(self.context, {'reservation_id': 'a', + 'image_id': 1, 'host': 'host2'}) comp1 = db.service_create(self.context, {'host': 'host1', 'availability_zone': 'zone1', @@ -222,11 +225,9 @@ class CloudTestCase(test.TestCase): db.service_destroy(self.context, comp2['id']) def test_console_output(self): - image_id = FLAGS.default_image - print image_id instance_type = FLAGS.default_instance_type max_count = 1 - kwargs = {'image_id': image_id, + kwargs = {'image_id': 'ami-1', 'instance_type': instance_type, 'max_count': max_count} rv = self.cloud.run_instances(self.context, **kwargs) @@ -242,8 +243,7 @@ class CloudTestCase(test.TestCase): greenthread.sleep(0.3) def test_ajax_console(self): - image_id = FLAGS.default_image - kwargs = {'image_id': image_id} + kwargs = {'image_id': 'ami-1'} rv = self.cloud.run_instances(self.context, **kwargs) instance_id = rv['instancesSet'][0]['instanceId'] greenthread.sleep(0.3) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 1f49baaf6..8c18fafc6 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -47,16 +47,16 @@ class ComputeTestCase(test.TestCase): network_manager='nova.network.manager.FlatManager') self.compute = utils.import_object(FLAGS.compute_manager) self.compute_api = compute.API() - - def fake_image_show(meh, context, id): - return dict(kernelId=1, ramdiskId=1) - - self.stubs.Set(local.LocalImageService, 'show', fake_image_show) self.manager = manager.AuthManager() self.user = self.manager.create_user('fake', 'fake', 'fake') self.project = self.manager.create_project('fake', 'fake', 'fake') self.context = context.RequestContext('fake', 'fake', False) + def fake_show(meh, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}} + + self.stubs.Set(local.LocalImageService, 'show', fake_show) + def tearDown(self): self.manager.delete_user(self.user) self.manager.delete_project(self.project) @@ -65,7 +65,7 @@ class ComputeTestCase(test.TestCase): def _create_instance(self): """Create a test instance""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' inst['user_id'] = self.user.id diff --git a/nova/tests/test_console.py b/nova/tests/test_console.py index 49ff24413..d47c70d88 100644 --- a/nova/tests/test_console.py +++ b/nova/tests/test_console.py @@ -57,7 +57,7 @@ class ConsoleTestCase(test.TestCase): inst = {} #inst['host'] = self.host #inst['name'] = 'instance-1234' - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' inst['user_id'] = self.user.id diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index ca8abdb36..45b544753 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -20,11 +20,12 @@ from nova import compute from nova import context from nova import db from nova import flags +from nova import network from nova import quota from nova import test from nova import utils +from nova import volume from nova.auth import manager -from nova.api.ec2 import cloud from nova.compute import instance_types @@ -41,7 +42,6 @@ class QuotaTestCase(test.TestCase): quota_gigabytes=20, quota_floating_ips=1) - self.cloud = cloud.CloudController() self.manager = manager.AuthManager() self.user = self.manager.create_user('admin', 'admin', 'admin', True) self.project = self.manager.create_project('admin', 'admin', 'admin') @@ -57,7 +57,7 @@ class QuotaTestCase(test.TestCase): def _create_instance(self, cores=2): """Create a test instance""" inst = {} - inst['image_id'] = 'ami-1' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['user_id'] = self.user.id inst['project_id'] = self.project.id @@ -118,12 +118,12 @@ class QuotaTestCase(test.TestCase): for i in range(FLAGS.quota_instances): instance_id = self._create_instance() instance_ids.append(instance_id) - self.assertRaises(quota.QuotaError, self.cloud.run_instances, + self.assertRaises(quota.QuotaError, compute.API().create, self.context, min_count=1, max_count=1, instance_type='m1.small', - image_id='ami-1') + image_id=1) for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -131,12 +131,12 @@ class QuotaTestCase(test.TestCase): instance_ids = [] instance_id = self._create_instance(cores=4) instance_ids.append(instance_id) - self.assertRaises(quota.QuotaError, self.cloud.run_instances, + self.assertRaises(quota.QuotaError, compute.API().create, self.context, min_count=1, max_count=1, instance_type='m1.small', - image_id='ami-1') + image_id=1) for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -145,9 +145,12 @@ class QuotaTestCase(test.TestCase): for i in range(FLAGS.quota_volumes): volume_id = self._create_volume() volume_ids.append(volume_id) - self.assertRaises(quota.QuotaError, self.cloud.create_volume, - self.context, - size=10) + self.assertRaises(quota.QuotaError, + volume.API().create, + self.context, + size=10, + name='', + description='') for volume_id in volume_ids: db.volume_destroy(self.context, volume_id) @@ -156,9 +159,11 @@ class QuotaTestCase(test.TestCase): volume_id = self._create_volume(size=20) volume_ids.append(volume_id) self.assertRaises(quota.QuotaError, - self.cloud.create_volume, + volume.API().create, self.context, - size=10) + size=10, + name='', + description='') for volume_id in volume_ids: db.volume_destroy(self.context, volume_id) @@ -172,7 +177,8 @@ class QuotaTestCase(test.TestCase): # make an rpc.call, the test just finishes with OK. It # appears to be something in the magic inline callbacks # that is breaking. - self.assertRaises(quota.QuotaError, self.cloud.allocate_address, + self.assertRaises(quota.QuotaError, + network.API().allocate_floating_ip, self.context) db.floating_ip_destroy(context.get_admin_context(), address) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index b6888c4d2..bb279ac4b 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -155,7 +155,7 @@ class SimpleDriverTestCase(test.TestCase): def _create_instance(self, **kwargs): """Create a test instance""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['user_id'] = self.user.id inst['project_id'] = self.project.id @@ -169,8 +169,6 @@ class SimpleDriverTestCase(test.TestCase): def _create_volume(self): """Create a test volume""" vol = {} - vol['image_id'] = 'ami-test' - vol['reservation_id'] = 'r-fakeres' vol['size'] = 1 vol['availability_zone'] = 'test' return db.volume_create(self.context, vol)['id'] diff --git a/nova/tests/test_volume.py b/nova/tests/test_volume.py index b40ca004b..f698c85b5 100644 --- a/nova/tests/test_volume.py +++ b/nova/tests/test_volume.py @@ -99,7 +99,7 @@ class VolumeTestCase(test.TestCase): def test_run_attach_detach_volume(self): """Make sure volume can be attached and detached from instance.""" inst = {} - inst['image_id'] = 'ami-test' + inst['image_id'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' inst['user_id'] = 'fake' diff --git a/nova/virt/images.py b/nova/virt/images.py index 7a6fef330..06b3a8ecd 100644 --- a/nova/virt/images.py +++ b/nova/virt/images.py @@ -28,29 +28,32 @@ import time import urllib2 import urlparse +from nova import context from nova import flags from nova import log as logging from nova import utils from nova.auth import manager from nova.auth import signer -from nova.objectstore import image FLAGS = flags.FLAGS -flags.DEFINE_bool('use_s3', True, - 'whether to get images from s3 or use local copy') - LOG = logging.getLogger('nova.virt.images') -def fetch(image, path, user, project): - if FLAGS.use_s3: - f = _fetch_s3_image - else: - f = _fetch_local_image - return f(image, path, user, project) +def fetch(image_id, path, _user, _project): + # TODO(vish): Improve context handling and add owner and auth data + # when it is added to glance. Right now there is no + # auth checking in glance, so we assume that access was + # checked before we got here. + image_service = utils.import_object(FLAGS.image_service) + with open(path, "wb") as image_file: + elevated = context.get_admin_context() + metadata = image_service.get(elevated, image_id, image_file) + return metadata +# NOTE(vish): The methods below should be unnecessary, but I'm leaving +# them in case the glance client does not work on windows. def _fetch_image_no_curl(url, path, headers): request = urllib2.Request(url) for (k, v) in headers.iteritems(): @@ -110,6 +113,8 @@ def _image_path(path): return os.path.join(FLAGS.images_path, path) +# TODO(vish): xenapi should use the glance client code directly instead +# of retrieving the image using this method. def image_url(image): if FLAGS.image_service == "nova.image.glance.GlanceImageService": return "http://%s:%s/images/%s" % (FLAGS.glance_host, diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 9f7315c17..02a8208d5 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -579,21 +579,23 @@ class LibvirtConnection(object): 'ramdisk_id': inst['ramdisk_id']} if disk_images['kernel_id']: + fname = '%08x' % int(disk_images['kernel_id']) self._cache_image(fn=self._fetch_image, target=basepath('kernel'), - fname=disk_images['kernel_id'], + fname=fname, image_id=disk_images['kernel_id'], user=user, project=project) if disk_images['ramdisk_id']: + fname = '%08x' % int(disk_images['ramdisk_id']) self._cache_image(fn=self._fetch_image, target=basepath('ramdisk'), - fname=disk_images['ramdisk_id'], + fname=fname, image_id=disk_images['ramdisk_id'], user=user, project=project) - root_fname = disk_images['image_id'] + root_fname = '%08x' % int(disk_images['image_id']) size = FLAGS.minimum_root_size if inst['instance_type'] == 'm1.tiny' or suffix == '.rescue': size = None -- cgit From 81fe66bb19d16c387705e144c9941096cc497cf0 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 7 Mar 2011 15:12:26 +0100 Subject: Make "dhcpbridge init" output correctly formatted leases information. --- nova/network/linux_net.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 535ce87bc..0bcc36081 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -18,6 +18,7 @@ Implements vlans, bridges, and iptables rules using linux utilities. """ import os +import time from nova import db from nova import exception @@ -56,6 +57,8 @@ flags.DEFINE_bool('use_nova_chains', False, 'use the nova_ routing chains instead of default') flags.DEFINE_string('input_chain', 'INPUT', 'chain to add nova_input to') +flags.DEFINE_integer('dhcp_lease_time', 120, + 'Lifetime of a DHCP lease') flags.DEFINE_string('dns_server', None, 'if set, uses specific dns server for dnsmasq') @@ -273,8 +276,17 @@ def ensure_bridge(bridge, interface, net_attrs=None): _confirm_rule("FORWARD", "-j nova-local") +def get_dhcp_leases(context, network_id): + """Return a network's hosts config in dnsmasq leasefile format""" + hosts = [] + for fixed_ip_ref in db.network_get_associated_fixed_ips(context, + network_id): + hosts.append(_host_lease(fixed_ip_ref)) + return '\n'.join(hosts) + + def get_dhcp_hosts(context, network_id): - """Get a string containing a network's hosts config in dnsmasq format""" + """Get a string containing a network's hosts config in dhcp-host format""" hosts = [] for fixed_ip_ref in db.network_get_associated_fixed_ips(context, network_id): @@ -365,8 +377,19 @@ interface %s utils.get_my_linklocal(network_ref['bridge'])}) +def _host_lease(fixed_ip_ref): + """Return a host string for an address in leasefile format""" + instance_ref = fixed_ip_ref['instance'] + timestamp = time.mktime(instance_ref['updated_at'].timetuple()) + + return "%d %s %s %s" % (timestamp + FLAGS.dhcp_lease_time, + instance_ref['mac_address'], + instance_ref['hostname'], + fixed_ip_ref['address']) + + def _host_dhcp(fixed_ip_ref): - """Return a host string for an address""" + """Return a host string for an address in dhcp-host format""" instance_ref = fixed_ip_ref['instance'] return "%s,%s.%s,%s" % (instance_ref['mac_address'], instance_ref['hostname'], @@ -420,7 +443,8 @@ def _dnsmasq_cmd(net): ' --pid-file=%s' % _dhcp_file(net['bridge'], 'pid'), ' --listen-address=%s' % net['gateway'], ' --except-interface=lo', - ' --dhcp-range=%s,static,120s' % net['dhcp_start'], + ' --dhcp-range=%s,static,%ds' % (net['dhcp_start'], + FLAGS.dhcp_lease_time), ' --dhcp-hostsfile=%s' % _dhcp_file(net['bridge'], 'conf'), ' --dhcp-script=%s' % FLAGS.dhcpbridge, ' --leasefile-ro'] -- cgit From b3d3366b8fd4eaf81bb9e03ad808c1a139e5b5b0 Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Mon, 7 Mar 2011 12:07:23 -0500 Subject: Generate 'adminPass' and call set_password when creating servers. --- nova/api/openstack/servers.py | 10 +++++++--- nova/tests/api/openstack/test_servers.py | 8 +++++++- 2 files changed, 14 insertions(+), 4 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 08b95b46a..6cd8bb451 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -84,13 +84,11 @@ def _translate_detail_keys(inst): return dict(server=inst_dict) - def _translate_keys(inst): """ Coerces into dictionary format, excluding all model attributes save for id and name """ return dict(server=dict(id=inst['id'], name=inst['display_name'])) - class Controller(wsgi.Controller): """ The Server API controller for the OpenStack API """ @@ -178,7 +176,13 @@ class Controller(wsgi.Controller): key_data=key_pair['public_key'], metadata=metadata, onset_files=env.get('onset_files', [])) - return _translate_keys(instances[0]) + + server = _translate_keys(instances[0]) + password = "%s%s" % (server['server']['name'][:4], + utils.generate_password(12)) + server['server']['adminPass'] = password + self.compute_api.set_admin_password(context, server['server']['id']) + return server def update(self, req, id): """ Updates the server name or password """ diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 78beb7df9..16b48a13b 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -186,7 +186,7 @@ class ServersTest(test.TestCase): def test_create_instance(self): def instance_create(context, inst): - return {'id': '1', 'display_name': ''} + return {'id': '1', 'display_name': 'server_test'} def server_update(context, id, params): return instance_create(context, id) @@ -230,6 +230,12 @@ class ServersTest(test.TestCase): res = req.get_response(fakes.wsgi_app()) + server = json.loads(res.body)['server'] + self.assertEqual('serv', server['adminPass'][:4]) + self.assertEqual(16, len(server['adminPass'])) + self.assertEqual('server_test', server['name']) + self.assertEqual('1', server['id']) + self.assertEqual(res.status_int, 200) def test_update_no_body(self): -- cgit From bcb18ee3d0d095b616c0909c92a151a599d4e17f Mon Sep 17 00:00:00 2001 From: Naveed Massjouni Date: Mon, 7 Mar 2011 15:05:07 -0500 Subject: Invalid values for offset and limit params in http requests now return a 400 response with a useful message in the body. Also added and updated tests. --- nova/api/openstack/common.py | 11 +++++++---- nova/tests/api/openstack/test_common.py | 20 ++++---------------- nova/tests/api/openstack/test_servers.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 39 insertions(+), 20 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 9f85c5c8a..f7a9cc3f0 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -36,15 +36,18 @@ def limited(items, request, max_limit=1000): try: offset = int(request.GET.get('offset', 0)) except ValueError: - offset = 0 + raise webob.exc.HTTPBadRequest(_('offset param must be an integer')) try: limit = int(request.GET.get('limit', max_limit)) except ValueError: - limit = max_limit + raise webob.exc.HTTPBadRequest(_('limit param must be an integer')) - if offset < 0 or limit < 0: - raise webob.exc.HTTPBadRequest() + if limit < 0: + raise webob.exc.HTTPBadRequest(_('limit param must be positive')) + + if offset < 0: + raise webob.exc.HTTPBadRequest(_('offset param must be positive')) limit = min(max_limit, limit or max_limit) range_end = offset + limit diff --git a/nova/tests/api/openstack/test_common.py b/nova/tests/api/openstack/test_common.py index 92023362c..8f57c5b67 100644 --- a/nova/tests/api/openstack/test_common.py +++ b/nova/tests/api/openstack/test_common.py @@ -79,20 +79,14 @@ class LimiterTest(test.TestCase): Test offset key works with a blank offset. """ req = Request.blank('/?offset=') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium) - self.assertEqual(limited(self.large, req), self.large[:1000]) + self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) def test_limiter_offset_bad(self): """ Test offset key works with a BAD offset. """ req = Request.blank(u'/?offset=\u0020aa') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium) - self.assertEqual(limited(self.large, req), self.large[:1000]) + self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) def test_limiter_nothing(self): """ @@ -166,18 +160,12 @@ class LimiterTest(test.TestCase): """ Test a negative limit. """ - def _limit_large(): - limited(self.large, req, max_limit=2000) - req = Request.blank('/?limit=-3000') - self.assertRaises(webob.exc.HTTPBadRequest, _limit_large) + self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) def test_limiter_negative_offset(self): """ Test a negative offset. """ - def _limit_large(): - limited(self.large, req, max_limit=2000) - req = Request.blank('/?offset=-30') - self.assertRaises(webob.exc.HTTPBadRequest, _limit_large) + self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 78beb7df9..10fb2bafb 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -184,6 +184,34 @@ class ServersTest(test.TestCase): self.assertEqual(s.get('imageId', None), None) i += 1 + def test_get_servers_with_limit(self): + req = webob.Request.blank('/v1.0/servers?limit=3') + res = req.get_response(fakes.wsgi_app()) + servers = json.loads(res.body)['servers'] + self.assertEqual([s['id'] for s in servers], [0, 1, 2]) + + req = webob.Request.blank('/v1.0/servers?limit=aaa') + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 400) + self.assertTrue('limit' in res.body) + + def test_get_servers_with_offset(self): + req = webob.Request.blank('/v1.0/servers?offset=2') + res = req.get_response(fakes.wsgi_app()) + servers = json.loads(res.body)['servers'] + self.assertEqual([s['id'] for s in servers], [2, 3, 4]) + + req = webob.Request.blank('/v1.0/servers?offset=aaa') + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 400) + self.assertTrue('offset' in res.body) + + def test_get_servers_with_limit_and_offset(self): + req = webob.Request.blank('/v1.0/servers?limit=2&offset=1') + res = req.get_response(fakes.wsgi_app()) + servers = json.loads(res.body)['servers'] + self.assertEqual([s['id'] for s in servers], [1, 2]) + def test_create_instance(self): def instance_create(context, inst): return {'id': '1', 'display_name': ''} -- cgit From 8c3bc15c96c6a6f1c99d829337921f2645608410 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 7 Mar 2011 21:43:31 +0100 Subject: Make iptables rules class __ne__ just be inverted __eq__. --- nova/network/linux_net.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 8832f7c68..57e77f8d5 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -85,10 +85,7 @@ class IptablesRule(object): (self.wrap == other.wrap)) def __ne__(self, other): - return ((self.chain != other.chain) or - (self.rule != other.rule) or - (self.top != other.top) or - (self.wrap != other.wrap)) + return not self == other def __str__(self): if self.wrap: -- cgit From 7b7abe7e7a25c0cd07c64c34f69ce050c669cfc3 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 7 Mar 2011 21:54:25 +0100 Subject: Log failed command execution if there are more retry attempts left. --- nova/utils.py | 1 + 1 file changed, 1 insertion(+) (limited to 'nova') diff --git a/nova/utils.py b/nova/utils.py index 829adfb9e..80e5b6bbe 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -166,6 +166,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, if not attempts: raise else: + LOG.debug(_("%r failed. Retrying."), cmd) greenthread.sleep(random.randint(20, 200) / 100.0) -- cgit From 4e9c570fbf8b3987d556da085b61f159f32c16f1 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 7 Mar 2011 21:59:05 +0100 Subject: Use IptablesManager.semapahore from securitygroups driver to ensure we don't apply half a rule set. --- nova/virt/libvirt_conn.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index b900cb8eb..825bcb0d7 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1372,8 +1372,12 @@ class IptablesFirewallDriver(FirewallDriver): def refresh_security_group_rules(self, security_group): for instance in self.instances.values(): - self.remove_filters_for_instance(instance) - self.add_filters_for_instance(instance) + # We use the semaphore to make sure noone applies the rule set + # after we've yanked the existing rules but before we've put in + # the new ones. + with self.iptables.semaphore: + self.remove_filters_for_instance(instance) + self.add_filters_for_instance(instance) self.iptables.apply() def _security_group_chain_name(self, security_group_id): -- cgit From 0abd5bfecd279272e5fe1b0de04478909cd77010 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Mon, 7 Mar 2011 22:18:15 +0100 Subject: added network_get_by_cidr method to nova.db api --- nova/db/api.py | 7 +++++++ nova/db/sqlalchemy/api.py | 18 ++++++++++++++++++ 2 files changed, 25 insertions(+) (limited to 'nova') diff --git a/nova/db/api.py b/nova/db/api.py index d23f14a3c..c73796487 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -459,6 +459,10 @@ def network_associate(context, project_id): """Associate a free network to a project.""" return IMPL.network_associate(context, project_id) +def network_is_associated(context, project_id): + """Returns true the the network is associated to a project""" + return IMPL.network_is_associated(context, project_id) + def network_count(context): """Return the number of networks.""" @@ -525,6 +529,9 @@ def network_get_by_bridge(context, bridge): """Get a network by bridge or raise if it does not exist.""" return IMPL.network_get_by_bridge(context, bridge) +def network_get_by_cidr(context, cidr): + """Get a network by cidr or raise if it does not exist""" + return IMPL.network_get_by_cidr(context, cidr) def network_get_by_instance(context, instance_id): """Get a network by instance id or raise if it does not exist.""" diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 919dda118..bd2de70c7 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -993,6 +993,13 @@ def network_associate(context, project_id): return network_ref +@require_admin_context +def network_is_associated(context, project_id): + session = get_session() + network = session.query(models.Network.project_id).filter(project_id=1).first() + print network + + @require_admin_context def network_count(context): session = get_session() @@ -1116,6 +1123,17 @@ def network_get_by_bridge(context, bridge): return result +@require_admin_context +def network_get_by_cidr(context, cidr): + session = get_session() + result = session.query(models.Network).\ + filter_by(cidr=cidr).first() + + if not result: + raise exception.NotFound(_('Network with cidr %s does not exist') % + cidr) + return result.id + @require_admin_context def network_get_by_instance(_context, instance_id): session = get_session() -- cgit From fd95523689b80f53972c59c3738e6b786a7160ff Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 7 Mar 2011 21:22:06 +0000 Subject: pep8 --- nova/api/ec2/cloud.py | 1 - nova/api/ec2/ec2utils.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 496e944fe..6479c9445 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -864,7 +864,6 @@ class CloudController(object): except exception.NotFound: return self.image_service.show_by_name(context, ec2_id) - def _format_image(self, image): """Convert from format defined by BaseImageService to S3 format.""" i = {} diff --git a/nova/api/ec2/ec2utils.py b/nova/api/ec2/ec2utils.py index e4df80cf8..3b34f6ea5 100644 --- a/nova/api/ec2/ec2utils.py +++ b/nova/api/ec2/ec2utils.py @@ -18,6 +18,7 @@ from nova import exception + def ec2_id_to_id(ec2_id): """Convert an ec2 ID (i-[base 16 number]) to an instance id (int)""" try: -- cgit From 56ee811efd52d0971d7fea4c232a904b3ee78ac6 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Mon, 7 Mar 2011 22:37:26 +0100 Subject: deleted network_is_associated from nova.db api --- nova/db/api.py | 5 ----- nova/db/sqlalchemy/api.py | 9 +-------- 2 files changed, 1 insertion(+), 13 deletions(-) (limited to 'nova') diff --git a/nova/db/api.py b/nova/db/api.py index c73796487..04f5fd72f 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -459,11 +459,6 @@ def network_associate(context, project_id): """Associate a free network to a project.""" return IMPL.network_associate(context, project_id) -def network_is_associated(context, project_id): - """Returns true the the network is associated to a project""" - return IMPL.network_is_associated(context, project_id) - - def network_count(context): """Return the number of networks.""" return IMPL.network_count(context) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index bd2de70c7..c8f42425d 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -993,13 +993,6 @@ def network_associate(context, project_id): return network_ref -@require_admin_context -def network_is_associated(context, project_id): - session = get_session() - network = session.query(models.Network.project_id).filter(project_id=1).first() - print network - - @require_admin_context def network_count(context): session = get_session() @@ -1132,7 +1125,7 @@ def network_get_by_cidr(context, cidr): if not result: raise exception.NotFound(_('Network with cidr %s does not exist') % cidr) - return result.id + return result @require_admin_context def network_get_by_instance(_context, instance_id): -- cgit From cac5881eaa35f94e004c18dd34ca78014f067976 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Tue, 8 Mar 2011 01:01:41 -0500 Subject: execvp --- nova/crypto.py | 32 ++++--- nova/network/linux_net.py | 222 ++++++++++++++++++++++--------------------- nova/tests/test_network.py | 16 ++-- nova/utils.py | 16 ++-- nova/virt/disk.py | 44 +++++---- nova/virt/images.py | 5 +- nova/virt/libvirt_conn.py | 36 ++++--- nova/virt/xenapi/vm_utils.py | 11 ++- nova/volume/driver.py | 71 +++++++------- 9 files changed, 238 insertions(+), 215 deletions(-) (limited to 'nova') diff --git a/nova/crypto.py b/nova/crypto.py index b240a3958..dd24723b8 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -105,8 +105,10 @@ def generate_key_pair(bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.join(tmpdir, 'temp') - utils.execute('ssh-keygen','-q','-b',"%d" % bits,'-N','""','-f',keyfile) - (out, err) = utils.execute('ssh-keygen','-q','-l','-f',"%s.pub" % (keyfile)) + utils.execute('ssh-keygen', '-q', '-b', '%d' % bits, '-N', '', + '-f', keyfile) + (out, err) = utils.execute('ssh-keygen', '-q', '-l', '-f', + '%s.pub' % (keyfile)) fingerprint = out.split(' ')[1] private_key = open(keyfile).read() public_key = open(keyfile + '.pub').read() @@ -118,7 +120,7 @@ def generate_key_pair(bits=1024): # bio = M2Crypto.BIO.MemoryBuffer() # key.save_pub_key_bio(bio) # public_key = bio.read() - # public_key, err = execute('ssh-keygen','-y','-f','/dev/stdin', private_key) + # public_key, err = execute('ssh-keygen', '-y', '-f', '/dev/stdin', private_key) return (private_key, public_key, fingerprint) @@ -143,9 +145,10 @@ def revoke_cert(project_id, file_name): start = os.getcwd() os.chdir(ca_folder(project_id)) # NOTE(vish): potential race condition here - utils.execute('openssl','ca','-config','./openssl.cnf','-revoke',"'%s'" % file_name) - utils.execute('openssl','ca','-gencrl','-config','./openssl.cnf','-out',"'%s'" % - FLAGS.crl_file) + utils.execute('openssl', 'ca', '-config', './openssl.cnf', '-revoke', + '%s' % file_name) + utils.execute('openssl', 'ca', '-gencrl', '-config', './openssl.cnf', + '-out', '%s' % FLAGS.crl_file) os.chdir(start) @@ -193,8 +196,9 @@ def generate_x509_cert(user_id, project_id, bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key')) csrfile = os.path.join(tmpdir, 'temp.csr') - utils.execute('openssl','genrsa','-out',keyfile,bits) - utils.execute('openssl','req','-new','-key',keyfile,'-out',csrfile,'-batch','-subj',subject) + utils.execute('openssl', 'genrsa', '-out', keyfile, bits) + utils.execute('openssl', 'req', '-new', '-key', keyfile, '-out', csrfile, + '-batch', '-subj', subject) private_key = open(keyfile).read() csr = open(csrfile).read() shutil.rmtree(tmpdir) @@ -211,7 +215,8 @@ def _ensure_project_folder(project_id): if not os.path.exists(ca_path(project_id)): start = os.getcwd() os.chdir(ca_folder()) - utils.execute('sh','geninter.sh',project_id, _project_cert_subject(project_id)) + utils.execute('sh', 'geninter.sh', project_id, + _project_cert_subject(project_id)) os.chdir(start) @@ -226,7 +231,7 @@ def generate_vpn_files(project_id): start = os.getcwd() os.chdir(ca_folder()) # TODO(vish): the shell scripts could all be done in python - utils.execute('sh','genvpn.sh', + utils.execute('sh', 'genvpn.sh', project_id, _vpn_cert_subject(project_id)) with open(csr_fn, "r") as csrfile: csr_text = csrfile.read() @@ -257,9 +262,10 @@ def _sign_csr(csr_text, ca_folder): start = os.getcwd() # Change working dir to CA os.chdir(ca_folder) - utils.execute('openssl','ca','-batch','-out',outbound,'-config' - './openssl.cnf','-infiles',inbound) - out, _err = utils.execute('openssl','x509','-in',outbound','-serial','-noout') + utils.execute('openssl', 'ca', '-batch', '-out', outbound, '-config', + './openssl.cnf', '-infiles', inbound) + out, _err = utils.execute('openssl', 'x509', '-in', outbound, + '-serial', '-noout') serial = out.rpartition("=")[2] os.chdir(start) with open(outbound, "r") as crtfile: diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 535ce87bc..ad019a8c0 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -65,113 +65,117 @@ flags.DEFINE_string('dmz_cidr', '10.128.0.0/24', def metadata_forward(): """Create forwarding rule for metadata""" - _confirm_rule("PREROUTING", "-t nat -s 0.0.0.0/0 " - "-d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT " - "--to-destination %s:%s" % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) + _confirm_rule("PREROUTING", '-t', 'nat', '-s', '0.0.0.0/0', + '-d', '169.254.169.254/32', '-p', 'tcp', '-m', 'tcp', + '--dport', '80', '-j', 'DNAT', + '--to-destination', '%s:%s' % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) def init_host(): """Basic networking setup goes here""" if FLAGS.use_nova_chains: - _execute("sudo iptables -N nova_input", check_exit_code=False) - _execute("sudo iptables -D %s -j nova_input" % FLAGS.input_chain, + _execute('sudo', 'iptables', '-N', 'nova_input', check_exit_code=False) + _execute('sudo', 'iptables', '-D', FLAGS.input_chain, + '-j', 'nova_input', check_exit_code=False) - _execute("sudo iptables -A %s -j nova_input" % FLAGS.input_chain) - - _execute("sudo iptables -N nova_forward", check_exit_code=False) - _execute("sudo iptables -D FORWARD -j nova_forward", + _execute('sudo', 'iptables', '-A', FLAGS.input_chain, + '-j', 'nova_input') + _execute('sudo', 'iptables', '-N', 'nova_forward', check_exit_code=False) - _execute("sudo iptables -A FORWARD -j nova_forward") - - _execute("sudo iptables -N nova_output", check_exit_code=False) - _execute("sudo iptables -D OUTPUT -j nova_output", + _execute('sudo', 'iptables', '-D', 'FORWARD', '-j', 'nova_forward', check_exit_code=False) - _execute("sudo iptables -A OUTPUT -j nova_output") - - _execute("sudo iptables -t nat -N nova_prerouting", + _execute('sudo', 'iptables', '-A', 'FORWARD', '-j', 'nova_forward') + _execute('sudo', 'iptables', '-N', 'nova_output', check_exit_code=False) + _execute('sudo', 'iptables', '-D', 'OUTPUT', '-j', 'nova_output', check_exit_code=False) - _execute("sudo iptables -t nat -D PREROUTING -j nova_prerouting", + _execute('sudo', 'iptables', '-A', 'OUTPUT', '-j', 'nova_output') + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'nova_prerouting', check_exit_code=False) - _execute("sudo iptables -t nat -A PREROUTING -j nova_prerouting") - - _execute("sudo iptables -t nat -N nova_postrouting", + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'PREROUTING', + '-j', 'nova_prerouting', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'PREROUTING', + '-j', 'nova_prerouting') + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'nova_postrouting', check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j nova_postrouting", + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'POSTROUTING', + '-j', 'nova_postrouting', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'POSTROUTING', + '-j', 'nova_postrouting') + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'nova_snatting', check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j nova_postrouting") - - _execute("sudo iptables -t nat -N nova_snatting", + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'POSTROUTING', + '-j nova_snatting', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'POSTROUTING', + '-j', 'nova_snatting') + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'nova_output', check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j nova_snatting", - check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j nova_snatting") - - _execute("sudo iptables -t nat -N nova_output", check_exit_code=False) - _execute("sudo iptables -t nat -D OUTPUT -j nova_output", - check_exit_code=False) - _execute("sudo iptables -t nat -A OUTPUT -j nova_output") + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'OUTPUT', + '-j nova_output', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'OUTPUT', + '-j', 'nova_output') else: # NOTE(vish): This makes it easy to ensure snatting rules always # come after the accept rules in the postrouting chain - _execute("sudo iptables -t nat -N SNATTING", - check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j SNATTING", + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'SNATTING', check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j SNATTING") + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'POSTROUTING', + '-j', 'SNATTING', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'POSTROUTING', + '-j', 'SNATTING') # NOTE(devcamcar): Cloud public SNAT entries and the default # SNAT rule for outbound traffic. - _confirm_rule("SNATTING", "-t nat -s %s " - "-j SNAT --to-source %s" - % (FLAGS.fixed_range, FLAGS.routing_source_ip), append=True) + _confirm_rule("SNATTING", '-t', 'nat', '-s', FLAGS.fixed_range, + '-j', 'SNAT', '--to-source', FLAGS.routing_source_ip, + append=True) - _confirm_rule("POSTROUTING", "-t nat -s %s -d %s -j ACCEPT" % - (FLAGS.fixed_range, FLAGS.dmz_cidr)) - _confirm_rule("POSTROUTING", "-t nat -s %(range)s -d %(range)s -j ACCEPT" % - {'range': FLAGS.fixed_range}) + _confirm_rule("POSTROUTING", '-t', 'nat', '-s', FLAGS.fixed_range, + '-d', FLAGS.dmz_cidr, '-j', 'ACCEPT') + _confirm_rule("POSTROUTING", '-t', 'nat', '-s', FLAGS.fixed_range, + '-d', FLAGS.fixed_range, '-j', 'ACCEPT') def bind_floating_ip(floating_ip, check_exit_code=True): """Bind ip to public interface""" - _execute("sudo ip addr add %s dev %s" % (floating_ip, - FLAGS.public_interface), + _execute('sudo', 'ip', 'addr', 'add', floating_ip, + 'dev', FLAGS.public_interface), check_exit_code=check_exit_code) def unbind_floating_ip(floating_ip): """Unbind a public ip from public interface""" - _execute("sudo ip addr del %s dev %s" % (floating_ip, - FLAGS.public_interface)) + _execute('sudo', 'ip', 'addr', 'del', floating_ip, + 'dev', FLAGS.public_interface)) def ensure_vlan_forward(public_ip, port, private_ip): """Sets up forwarding rules for vlan""" - _confirm_rule("FORWARD", "-d %s -p udp --dport 1194 -j ACCEPT" % - private_ip) - _confirm_rule("PREROUTING", - "-t nat -d %s -p udp --dport %s -j DNAT --to %s:1194" - % (public_ip, port, private_ip)) + _confirm_rule("FORWARD", '-d', private_ip, '-p', 'udp', + '--dport', '1194', '-j', 'ACCEPT') + _confirm_rule("PREROUTING", '-t', 'nat', '-d', public_ip, '-p', 'udp', + '--dport', port, '-j', 'DNAT', '--to', '%s:1194' + % private_ip) def ensure_floating_forward(floating_ip, fixed_ip): """Ensure floating ip forwarding rule""" - _confirm_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _confirm_rule("OUTPUT", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _confirm_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" - % (fixed_ip, floating_ip)) + _confirm_rule("PREROUTING", '-t', 'nat', '-d', floating_ip, '-j', 'DNAT', + '--to', fixed_ip) + _confirm_rule("OUTPUT", '-t', 'nat', '-d', floating_ip, '-j', 'DNAT', + '--to', fixed_ip) + _confirm_rule("SNATTING", '-t', 'nat', '-s', fixed_ip, '-j', 'SNAT', + '--to', floating_ip) def remove_floating_forward(floating_ip, fixed_ip): """Remove forwarding for floating ip""" - _remove_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _remove_rule("OUTPUT", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _remove_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" - % (fixed_ip, floating_ip)) + _remove_rule("PREROUTING", '-t', 'nat', '-d', floating_ip, '-j', 'DNAT', + '--to', fixed_ip) + _remove_rule("OUTPUT", '-t', 'nat', '-d', floating_ip, '-j', 'DNAT', + '--to', fixed_ip) + _remove_rule("SNATTING", '-t', 'nat', '-s', fixed_ip, '-j', 'SNAT', + '--to', floating_ip) def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): @@ -185,9 +189,9 @@ def ensure_vlan(vlan_num): interface = "vlan%s" % vlan_num if not _device_exists(interface): LOG.debug(_("Starting VLAN inteface %s"), interface) - _execute("sudo vconfig set_name_type VLAN_PLUS_VID_NO_PAD") - _execute("sudo vconfig add %s %s" % (FLAGS.vlan_interface, vlan_num)) - _execute("sudo ip link set %s up" % interface) + _execute('sudo', 'vconfig', 'set_name_type', 'VLAN_PLUS_VID_NO_PAD') + _execute('sudo', 'vconfig', 'add', FLAGS.vlan_interface, vlan_num) + _execute('sudo', 'ip', 'link', 'set', interface, 'up') return interface @@ -206,52 +210,54 @@ def ensure_bridge(bridge, interface, net_attrs=None): """ if not _device_exists(bridge): LOG.debug(_("Starting Bridge interface for %s"), interface) - _execute("sudo brctl addbr %s" % bridge) - _execute("sudo brctl setfd %s 0" % bridge) + _execute('sudo', 'brctl', 'addbr', bridge) + _execute('sudo', 'brctl', 'setfd', bridge, 0) # _execute("sudo brctl setageing %s 10" % bridge) - _execute("sudo brctl stp %s off" % bridge) - _execute("sudo ip link set %s up" % bridge) + _execute('sudo', 'brctl', 'stp', bridge', 'off') + _execute('sudo', 'ip', 'link', 'set', bridge, up) if net_attrs: # NOTE(vish): The ip for dnsmasq has to be the first address on the # bridge for it to respond to reqests properly suffix = net_attrs['cidr'].rpartition('/')[2] - out, err = _execute("sudo ip addr add %s/%s brd %s dev %s" % - (net_attrs['gateway'], - suffix, - net_attrs['broadcast'], - bridge), + out, err = _execute('sudo', 'ip', 'addr', 'add', + "%s/%s" % + (net_attrs['gateway'], suffix), + 'brd', + net-attrs['broadcast'], + 'dev', + bridge, check_exit_code=False) if err and err != "RTNETLINK answers: File exists\n": raise exception.Error("Failed to add ip: %s" % err) if(FLAGS.use_ipv6): - _execute("sudo ip -f inet6 addr change %s dev %s" % - (net_attrs['cidr_v6'], bridge)) + _execute('sudo', 'ip', '-f', 'inet6', 'addr', + 'change', net_attrs['cidr_v6'], + 'dev', bridge) # NOTE(vish): If the public interface is the same as the # bridge, then the bridge has to be in promiscuous # to forward packets properly. if(FLAGS.public_interface == bridge): - _execute("sudo ip link set dev %s promisc on" % bridge) + _execute('sudo', 'ip', 'link', 'set', 'dev', bridge, 'promisc', 'on') if interface: # NOTE(vish): This will break if there is already an ip on the # interface, so we move any ips to the bridge gateway = None - out, err = _execute("sudo route -n") + out, err = _execute('sudo', 'route', '-n') for line in out.split("\n"): fields = line.split() if fields and fields[0] == "0.0.0.0" and fields[-1] == interface: gateway = fields[1] - out, err = _execute("sudo ip addr show dev %s scope global" % - interface) + out, err = _execute('sudo', 'ip', 'addr', 'show', 'dev', interface, + 'scope', 'global') for line in out.split("\n"): fields = line.split() if fields and fields[0] == "inet": params = ' '.join(fields[1:-1]) - _execute("sudo ip addr del %s dev %s" % (params, fields[-1])) - _execute("sudo ip addr add %s dev %s" % (params, bridge)) + _execute('sudo', 'ip', 'addr', 'del', params, 'dev', fields[-1]) + _execute('sudo', 'ip', 'addr', 'add', params, 'dev', bridge) if gateway: - _execute("sudo route add 0.0.0.0 gw %s" % gateway) - out, err = _execute("sudo brctl addif %s %s" % - (bridge, interface), + _execute('sudo', 'route', 'add', '0.0.0.0', 'gw', gateway) + out, err = _execute('sudo', 'brctl', 'addif, bridge, interface, check_exit_code=False) if (err and err != "device %s is already a member of a bridge; can't " @@ -259,18 +265,18 @@ def ensure_bridge(bridge, interface, net_attrs=None): raise exception.Error("Failed to add interface: %s" % err) if FLAGS.use_nova_chains: - (out, err) = _execute("sudo iptables -N nova_forward", + (out, err) = _execute('sudo', 'iptables', '-N', 'nova_forward, check_exit_code=False) if err != 'iptables: Chain already exists.\n': # NOTE(vish): chain didn't exist link chain - _execute("sudo iptables -D FORWARD -j nova_forward", + _execute('sudo', 'iptables, '-D', 'FORWARD', '-j', 'nova_forward', check_exit_code=False) - _execute("sudo iptables -A FORWARD -j nova_forward") + _execute('sudo', 'iptables', '-A', 'FORWARD', '-j', 'nova_forward') - _confirm_rule("FORWARD", "--in-interface %s -j ACCEPT" % bridge) - _confirm_rule("FORWARD", "--out-interface %s -j ACCEPT" % bridge) - _execute("sudo iptables -N nova-local", check_exit_code=False) - _confirm_rule("FORWARD", "-j nova-local") + _confirm_rule("FORWARD", '--in-interface', bridge, '-j', 'ACCEPT') + _confirm_rule("FORWARD", '--out-interface', bridge, '-j', 'ACCEPT') + _execute('sudo', 'iptables', '-N', 'nova-local', check_exit_code=False) + _confirm_rule("FORWARD", '-j', 'nova-local') def get_dhcp_hosts(context, network_id): @@ -304,11 +310,11 @@ def update_dhcp(context, network_id): # if dnsmasq is already running, then tell it to reload if pid: - out, _err = _execute('cat /proc/%d/cmdline' % pid, + out, _err = _execute('cat', "/proc/%d/cmdline" % pid, check_exit_code=False) if conffile in out: try: - _execute('sudo kill -HUP %d' % pid) + _execute('sudo', 'kill', '-HUP', pid) return except Exception as exc: # pylint: disable-msg=W0703 LOG.debug(_("Hupping dnsmasq threw %s"), exc) @@ -349,11 +355,11 @@ interface %s # if radvd is already running, then tell it to reload if pid: - out, _err = _execute('cat /proc/%d/cmdline' + out, _err = _execute('cat', "/proc/%d/cmdline' % pid, check_exit_code=False) if conffile in out: try: - _execute('sudo kill %d' % pid) + _execute('sudo', 'kill', pid) except Exception as exc: # pylint: disable-msg=W0703 LOG.debug(_("killing radvd threw %s"), exc) else: @@ -374,23 +380,23 @@ def _host_dhcp(fixed_ip_ref): fixed_ip_ref['address']) -def _execute(cmd, *args, **kwargs): +def _execute(*cmd, **kwargs): """Wrapper around utils._execute for fake_network""" if FLAGS.fake_network: - LOG.debug("FAKE NET: %s", cmd) + LOG.debug("FAKE NET: %s", ' '.join(cmd)) return "fake", 0 else: - return utils.execute(cmd, *args, **kwargs) + return utils.execute(*cmd, **kwargs) def _device_exists(device): """Check if ethernet device exists""" - (_out, err) = _execute("ip link show dev %s" % device, + (_out, err) = _execute('ip', 'link', 'show', 'dev', device, check_exit_code=False) return not err -def _confirm_rule(chain, cmd, append=False): +def _confirm_rule(chain, *cmd, append=False): """Delete and re-add iptables rule""" if FLAGS.use_nova_chains: chain = "nova_%s" % chain.lower() @@ -398,16 +404,16 @@ def _confirm_rule(chain, cmd, append=False): loc = "-A" else: loc = "-I" - _execute("sudo iptables --delete %s %s" % (chain, cmd), + _execute('sudo', 'iptables', '--delete', chain, *cmd, check_exit_code=False) - _execute("sudo iptables %s %s %s" % (loc, chain, cmd)) + _execute('sudo', 'iptables', loc, chain, *cmd) -def _remove_rule(chain, cmd): +def _remove_rule(chain, *cmd): """Remove iptables rule""" if FLAGS.use_nova_chains: chain = "%s" % chain.lower() - _execute("sudo iptables --delete %s %s" % (chain, cmd)) + _execute('sudo', 'iptables', '--delete', chain, *cmd) def _dnsmasq_cmd(net): @@ -444,7 +450,7 @@ def _stop_dnsmasq(network): if pid: try: - _execute('sudo kill -TERM %d' % pid) + _execute('sudo', 'kill', '-TERM', pid) except Exception as exc: # pylint: disable-msg=W0703 LOG.debug(_("Killing dnsmasq threw %s"), exc) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index ce1c77210..6d2d8b771 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -343,13 +343,13 @@ def lease_ip(private_ip): private_ip) instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) - cmd = "%s add %s %s fake" % (binpath('nova-dhcpbridge'), - instance_ref['mac_address'], - private_ip) + cmd = (binpath('nova-dhcpbridge'), 'add' + instance_ref['mac_address'], + private_ip, 'fake') env = {'DNSMASQ_INTERFACE': network_ref['bridge'], 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} - (out, err) = utils.execute(cmd, addl_env=env) + (out, err) = utils.execute(*cmd, addl_env=env) LOG.debug("ISSUE_IP: %s, %s ", out, err) @@ -359,11 +359,11 @@ def release_ip(private_ip): private_ip) instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) - cmd = "%s del %s %s fake" % (binpath('nova-dhcpbridge'), - instance_ref['mac_address'], - private_ip) + cmd = (binpath('nova-dhcpbridge'), 'del', + instance_ref['mac_address'], + private_ip, 'fake') env = {'DNSMASQ_INTERFACE': network_ref['bridge'], 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} - (out, err) = utils.execute(cmd, addl_env=env) + (out, err) = utils.execute(*cmd, addl_env=env) LOG.debug("RELEASE_IP: %s, %s ", out, err) diff --git a/nova/utils.py b/nova/utils.py index 40a8d8d8c..c96b85294 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -125,15 +125,15 @@ def fetchfile(url, target): # c.perform() # c.close() # fp.close() - execute("curl","--fail",url,"-o",target) + execute("curl", "--fail", url, "-o", target) -def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (subprocess): %s"), cmd) +def execute(*cmd, process_input=None, addl_env=None, check_exit_code=True): + LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(cmd, stdin=subprocess.PIPE, + obj = subprocess.Popen(*cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) result = None if process_input != None: @@ -148,7 +148,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, stderr=stderr, - cmd=cmd) + cmd=' '.join(cmd)) # NOTE(termie): this appears to be necessary to let the subprocess call # clean something up in between calls, without it two # execute calls in a row hangs the second one @@ -158,7 +158,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): def ssh_execute(ssh, cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (SSH): %s"), cmd) + LOG.debug(_("Running cmd (SSH): %s"), ' '.join(cmd)) if addl_env: raise exception.Error("Environment not supported over SSH") @@ -187,7 +187,7 @@ def ssh_execute(ssh, cmd, process_input=None, raise exception.ProcessExecutionError(exit_code=exit_status, stdout=stdout, stderr=stderr, - cmd=cmd) + cmd=' '.join(cmd)) return (stdout, stderr) @@ -254,7 +254,7 @@ def last_octet(address): def get_my_linklocal(interface): try: - if_str = execute("ip","-f","inet6","-o","addr","show", interface) + if_str = execute("ip", "-f", "inet6", "-o", "addr", "show", interface) condition = "\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link" links = [re.search(condition, x) for x in if_str[0].split('\n')] address = [w.group(1) for w in links if w is not None] diff --git a/nova/virt/disk.py b/nova/virt/disk.py index 2bded07a4..203517275 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -49,10 +49,10 @@ def extend(image, size): file_size = os.path.getsize(image) if file_size >= size: return - utils.execute('truncate -s %s %s' % (size, image)) + utils.execute('truncate', '-s', size, image) # NOTE(vish): attempts to resize filesystem - utils.execute('e2fsck -fp %s' % image, check_exit_code=False) - utils.execute('resize2fs %s' % image, check_exit_code=False) + utils.execute('e2fsck', '-fp', mage, check_exit_code=False) + utils.execute('resize2fs', image, check_exit_code=False) def inject_data(image, key=None, net=None, partition=None, nbd=False): @@ -68,7 +68,7 @@ def inject_data(image, key=None, net=None, partition=None, nbd=False): try: if not partition is None: # create partition - out, err = utils.execute('sudo kpartx -a %s' % device) + out, err = utils.execute('sudo', 'kpartx', '-a', device) if err: raise exception.Error(_('Failed to load partition: %s') % err) mapped_device = '/dev/mapper/%sp%s' % (device.split('/')[-1], @@ -84,13 +84,14 @@ def inject_data(image, key=None, net=None, partition=None, nbd=False): mapped_device) # Configure ext2fs so that it doesn't auto-check every N boots - out, err = utils.execute('sudo tune2fs -c 0 -i 0 %s' % mapped_device) + out, err = utils.execute('sudo', 'tune2fs', + '-c', 0, '-i', 0, mapped_device) tmpdir = tempfile.mkdtemp() try: # mount loopback to dir out, err = utils.execute( - 'sudo mount %s %s' % (mapped_device, tmpdir)) + 'sudo', 'mount', mapped_device, tmpdir) if err: raise exception.Error(_('Failed to mount filesystem: %s') % err) @@ -103,13 +104,13 @@ def inject_data(image, key=None, net=None, partition=None, nbd=False): _inject_net_into_fs(net, tmpdir) finally: # unmount device - utils.execute('sudo umount %s' % mapped_device) + utils.execute('sudo', 'umount', mapped_device) finally: # remove temporary directory - utils.execute('rmdir %s' % tmpdir) + utils.execute('rmdir', tmpdir) if not partition is None: # remove partitions - utils.execute('sudo kpartx -d %s' % device) + utils.execute('sudo', 'kpartx', '-d', device) finally: _unlink_device(device, nbd) @@ -118,7 +119,7 @@ def _link_device(image, nbd): """Link image to device using loopback or nbd""" if nbd: device = _allocate_device() - utils.execute('sudo qemu-nbd -c %s %s' % (device, image)) + utils.execute('sudo', 'qemu-nbd', '-c', device, image) # NOTE(vish): this forks into another process, so give it a chance # to set up before continuuing for i in xrange(FLAGS.timeout_nbd): @@ -127,7 +128,7 @@ def _link_device(image, nbd): time.sleep(1) raise exception.Error(_('nbd device %s did not show up') % device) else: - out, err = utils.execute('sudo losetup --find --show %s' % image) + out, err = utils.execute('sudo', 'losetup', '--find', '--show', image) if err: raise exception.Error(_('Could not attach image to loopback: %s') % err) @@ -137,10 +138,10 @@ def _link_device(image, nbd): def _unlink_device(device, nbd): """Unlink image from device using loopback or nbd""" if nbd: - utils.execute('sudo qemu-nbd -d %s' % device) + utils.execute('sudo', 'qemu-nbd', '-d', device) _free_device(device) else: - utils.execute('sudo losetup --detach %s' % device) + utils.execute('sudo', 'losetup', '--detach', device) _DEVICES = ['/dev/nbd%s' % i for i in xrange(FLAGS.max_nbd_devices)] @@ -170,11 +171,12 @@ def _inject_key_into_fs(key, fs): fs is the path to the base of the filesystem into which to inject the key. """ sshdir = os.path.join(fs, 'root', '.ssh') - utils.execute('sudo mkdir -p %s' % sshdir) # existing dir doesn't matter - utils.execute('sudo chown root %s' % sshdir) - utils.execute('sudo chmod 700 %s' % sshdir) + utils.execute('sudo', 'mkdir', '-p', sshdir) # existing dir doesn't matter + utils.execute('sudo', 'chown', 'root', sshdir) + utils.execute('sudo', 'chmod', '700', sshdir) keyfile = os.path.join(sshdir, 'authorized_keys') - utils.execute('sudo tee -a %s' % keyfile, '\n' + key.strip() + '\n') + # TODO:EWINDISCH: not sure about the following /w execv patch + utils.execute('sudo', 'tee', '-a', keyfile, '\n' + key.strip() + '\n') def _inject_net_into_fs(net, fs): @@ -183,8 +185,8 @@ def _inject_net_into_fs(net, fs): net is the contents of /etc/network/interfaces. """ netdir = os.path.join(os.path.join(fs, 'etc'), 'network') - utils.execute('sudo mkdir -p %s' % netdir) # existing dir doesn't matter - utils.execute('sudo chown root:root %s' % netdir) - utils.execute('sudo chmod 755 %s' % netdir) + utils.execute('sudo', 'mkdir', '-p', netdir) # existing dir doesn't matter + utils.execute('sudo', 'chown', 'root:root', netdir) + utils.execute('sudo', 'chmod', 755, netdir) netfile = os.path.join(netdir, 'interfaces') - utils.execute('sudo tee %s' % netfile, net) + utils.execute('sudo', 'tee', netfile, net) diff --git a/nova/virt/images.py b/nova/virt/images.py index 7a6fef330..4b11d1667 100644 --- a/nova/virt/images.py +++ b/nova/virt/images.py @@ -94,8 +94,7 @@ def _fetch_s3_image(image, path, user, project): cmd += ['-H', '\'%s: %s\'' % (k, v)] cmd += ['-o', path] - cmd_out = ' '.join(cmd) - return utils.execute(cmd_out) + return utils.execute(*cmd) def _fetch_local_image(image, path, user, project): @@ -103,7 +102,7 @@ def _fetch_local_image(image, path, user, project): if sys.platform.startswith('win'): return shutil.copy(source, path) else: - return utils.execute('cp %s %s' % (source, path)) + return utils.execute('cp', source, path) def _image_path(path): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 4e0fd106f..464ec475c 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -438,8 +438,10 @@ class LibvirtConnection(object): if virsh_output.startswith('/dev/'): LOG.info(_("cool, it's a device")) - out, err = utils.execute("sudo dd if=%s iflag=nonblock" % - virsh_output, check_exit_code=False) + out, err = utils.execute('sudo', 'dd', + "if=%s" % virsh_output, + 'iflag=nonblock', + check_exit_code=False) return out else: return '' @@ -461,11 +463,11 @@ class LibvirtConnection(object): console_log = os.path.join(FLAGS.instances_path, instance['name'], 'console.log') - utils.execute('sudo chown %d %s' % (os.getuid(), console_log)) + utils.execute('sudo', 'chown', s.getuid(), console_log) if FLAGS.libvirt_type == 'xen': # Xen is special - virsh_output = utils.execute("virsh ttyconsole %s" % + virsh_output = utils.execute('virsh', 'ttyconsole', instance['name']) data = self._flush_xen_console(virsh_output) fpath = self._append_to_file(data, console_log) @@ -482,7 +484,10 @@ class LibvirtConnection(object): port = random.randint(int(start_port), int(end_port)) # netcat will exit with 0 only if the port is in use, # so a nonzero return value implies it is unused - cmd = 'netcat 0.0.0.0 %s -w 1 Date: Tue, 8 Mar 2011 01:08:13 -0500 Subject: Fix todo comment --- nova/virt/libvirt_conn.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index e1cd75306..a5256bbc2 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -486,8 +486,9 @@ class LibvirtConnection(object): # netcat will exit with 0 only if the port is in use, # so a nonzero return value implies it is unused - # TODO:ewindisch: subprocess lets us do this... - # but utils.execute abstracts it away from us + # TODO(ewindisch): broken /w execvp patch. + # subprocess lets us do this, but utils.execute + # abstracts it away from us cmd = 'netcat', '0.0.0.0', port, '-w', '1', ' Date: Tue, 8 Mar 2011 14:35:53 +0000 Subject: * pep8 cleanups in migrations * a few bugfixes --- .../sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py | 5 +---- nova/tests/test_xenapi.py | 4 ++-- nova/virt/xenapi/vm_utils.py | 2 +- 3 files changed, 4 insertions(+), 7 deletions(-) (limited to 'nova') diff --git a/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py index a50f31e16..514b92b81 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py @@ -29,8 +29,6 @@ instances = Table('instances', meta, Column('id', Integer(), primary_key=True, nullable=False), ) -# FIXME(dubs) should this be not null? Maybe create as nullable, then -# populate all existing rows with 'linux', then adding not null constraint. instances_os_type = Column('os_type', String(length=255, convert_unicode=False, assert_unicode=None, unicode_error=None, @@ -45,7 +43,7 @@ def upgrade(migrate_engine): instances.create_column(instances_os_type) migrate_engine.execute(instances.update()\ - .where(instances.c.os_type==None)\ + .where(instances.c.os_type == None)\ .values(os_type='linux')) @@ -53,4 +51,3 @@ def downgrade(migrate_engine): meta.bind = migrate_engine instances.drop_column('os_type') - diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 27f0e5dd7..25070e108 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -251,7 +251,7 @@ class XenAPIVMTestCase(test.TestCase): self.vm_info = vm_info self.vm = vm - def check_vm_record(self): + def check_vm_record(self, conn): # Check that m1.large above turned into the right thing. instance_type = db.instance_type_get_by_name(conn, 'm1.large') mem_kib = long(instance_type['memory_mb']) << 10 @@ -321,7 +321,7 @@ class XenAPIVMTestCase(test.TestCase): instance = db.instance_create(values) conn.spawn(instance) self.create_vm_record(conn, os_type) - self.check_vm_record() + self.check_vm_record(conn) def test_spawn_not_enough_memory(self): FLAGS.xenapi_image_service = 'glance' diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 150824400..604e8a4e0 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -324,7 +324,7 @@ class VMHelper(HelperBase): kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'upload_vhd', kwargs) - session.wait_for_task(task, instance_id) + session.wait_for_task(task, instance.id) @classmethod def fetch_image(cls, session, instance_id, image, user, project, -- cgit From cbc2956a4e863c1bc952c7cef6045c39d293818d Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Tue, 8 Mar 2011 17:18:13 +0000 Subject: Remove addition of account to service url. --- nova/api/openstack/__init__.py | 24 +++------------- nova/api/openstack/auth.py | 46 ++++++----------------------- nova/auth/novarc.template | 2 +- nova/tests/api/openstack/test_accounts.py | 8 +++--- nova/tests/api/openstack/test_adminapi.py | 4 +-- nova/tests/api/openstack/test_auth.py | 2 +- nova/tests/api/openstack/test_flavors.py | 4 +-- nova/tests/api/openstack/test_images.py | 4 +-- nova/tests/api/openstack/test_servers.py | 48 +++++++++++++++---------------- nova/tests/api/openstack/test_users.py | 10 +++---- nova/tests/api/openstack/test_zones.py | 10 +++---- 11 files changed, 59 insertions(+), 103 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 005d330a6..a655b1c85 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -73,18 +73,6 @@ class APIRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() - accounts_controller = accounts.Controller() - mapper.connect("account", "/{id}", - controller=accounts_controller, action="show", - conditions=dict(method=["GET"])) - if FLAGS.allow_admin_api: - mapper.connect("/{id}", - controller=accounts_controller, action="update", - conditions=dict(method=["PUT"])) - mapper.connect("/{id}", - controller=accounts_controller, action="delete", - conditions=dict(method=["DELETE"])) - server_members = {'action': 'POST'} if FLAGS.allow_admin_api: LOG.debug(_("Including admin operations in API.")) @@ -101,38 +89,34 @@ class APIRouter(wsgi.Router): server_members['inject_network_info'] = 'POST' mapper.resource("zone", "zones", controller=zones.Controller(), - path_prefix="{account_id}/", collection={'detail': 'GET'}) mapper.resource("user", "users", controller=users.Controller(), - path_prefix="{account_id}/", collection={'detail': 'GET'}) + mapper.resource("account", "accounts", + controller=accounts.Controller(), + collection={'detail': 'GET'}) + mapper.resource("server", "servers", controller=servers.Controller(), collection={'detail': 'GET'}, - path_prefix="{account_id}/", member=server_members) mapper.resource("backup_schedule", "backup_schedule", controller=backup_schedules.Controller(), - path_prefix="{account_id}/servers/{server_id}/", parent_resource=dict(member_name='server', collection_name='servers')) mapper.resource("console", "consoles", controller=consoles.Controller(), - path_prefix="{account_id}/servers/{server_id}/", parent_resource=dict(member_name='server', collection_name='servers')) mapper.resource("image", "images", controller=images.Controller(), - path_prefix="{account_id}/", collection={'detail': 'GET'}) mapper.resource("flavor", "flavors", controller=flavors.Controller(), - path_prefix="{account_id}/", collection={'detail': 'GET'}) mapper.resource("shared_ip_group", "shared_ip_groups", - path_prefix="{account_id}/", collection={'detail': 'GET'}, controller=shared_ip_groups.Controller()) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index e77910fed..e71fc69e3 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -53,19 +53,15 @@ class AuthMiddleware(wsgi.Middleware): if not self.has_authentication(req): return self.authenticate(req) user = self.get_user_by_authentication(req) - account_name = req.path_info_peek() - + accounts = self.auth.get_projects(user=user) if not user: return faults.Fault(webob.exc.HTTPUnauthorized()) - if not account_name: - if self.auth.is_admin(user): - account_name = FLAGS.default_project - else: - return faults.Fault(webob.exc.HTTPUnauthorized()) - try: - account = self.auth.get_project(account_name) - except exception.NotFound: + if accounts: + #we are punting on this til auth is settled, + #and possibly til api v1.1 (mdragon) + account = accounts[0] + else: return faults.Fault(webob.exc.HTTPUnauthorized()) if not self.auth.is_admin(user) and \ @@ -85,7 +81,6 @@ class AuthMiddleware(wsgi.Middleware): # Unless the request is explicitly made against // don't # honor it path_info = req.path_info - account_name = None if len(path_info) > 1: return faults.Fault(webob.exc.HTTPUnauthorized()) @@ -95,10 +90,7 @@ class AuthMiddleware(wsgi.Middleware): except KeyError: return faults.Fault(webob.exc.HTTPUnauthorized()) - if ':' in username: - account_name, username = username.rsplit(':', 1) - - token, user = self._authorize_user(username, account_name, key, req) + token, user = self._authorize_user(username, key, req) if user and token: res = webob.Response() res.headers['X-Auth-Token'] = token.token_hash @@ -135,31 +127,15 @@ class AuthMiddleware(wsgi.Middleware): return self.auth.get_user(token.user_id) return None - def _authorize_user(self, username, account_name, key, req): + def _authorize_user(self, username, key, req): """Generates a new token and assigns it to a user. username - string - account_name - string key - string API key req - webob.Request object """ ctxt = context.get_admin_context() user = self.auth.get_user_from_access_key(key) - if account_name: - try: - account = self.auth.get_project(account_name) - except exception.NotFound: - return None, None - else: - # (dragondm) punt and try to determine account. - # this is something of a hack, but a user on 1 account is a - # common case, and is the way the current RS code works. - accounts = self.auth.get_projects(user=user) - if len(accounts) == 1: - account = accounts[0] - else: - #we can't tell what account they are logging in for. - return None, None if user and user.name == username: token_hash = hashlib.sha1('%s%s%f' % (username, key, @@ -167,11 +143,7 @@ class AuthMiddleware(wsgi.Middleware): token_dict = {} token_dict['token_hash'] = token_hash token_dict['cdn_management_url'] = '' - # auth url + project (account) id, e.g. - # http://foo.org:8774/baz/v1.0/myacct/ - os_url = '%s%s%s/' % (req.url, - '' if req.url.endswith('/') else '/', - account.id) + os_url = req.url token_dict['server_management_url'] = os_url token_dict['storage_url'] = '' token_dict['user_id'] = user.id diff --git a/nova/auth/novarc.template b/nova/auth/novarc.template index 1c917ad44..cda2ecc28 100644 --- a/nova/auth/novarc.template +++ b/nova/auth/novarc.template @@ -11,5 +11,5 @@ export EUCALYPTUS_CERT=${NOVA_CERT} # euca-bundle-image seems to require this se alias ec2-bundle-image="ec2-bundle-image --cert ${EC2_CERT} --privatekey ${EC2_PRIVATE_KEY} --user 42 --ec2cert ${NOVA_CERT}" alias ec2-upload-bundle="ec2-upload-bundle -a ${EC2_ACCESS_KEY} -s ${EC2_SECRET_KEY} --url ${S3_URL} --ec2cert ${NOVA_CERT}" export NOVA_API_KEY="%(access)s" -export NOVA_USERNAME="%(project)s:%(user)s" +export NOVA_USERNAME="%(user)s" export NOVA_URL="%(os)s" diff --git a/nova/tests/api/openstack/test_accounts.py b/nova/tests/api/openstack/test_accounts.py index b2e89824a..746f02f57 100644 --- a/nova/tests/api/openstack/test_accounts.py +++ b/nova/tests/api/openstack/test_accounts.py @@ -70,7 +70,7 @@ class AccountsTest(test.TestCase): super(AccountsTest, self).tearDown() def test_get_account(self): - req = webob.Request.blank('/v1.0/test1') + req = webob.Request.blank('/v1.0/accounts/test1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -80,7 +80,7 @@ class AccountsTest(test.TestCase): self.assertEqual(res.status_int, 200) def test_account_delete(self): - req = webob.Request.blank('/v1.0/test1') + req = webob.Request.blank('/v1.0/accounts/test1') req.method = 'DELETE' res = req.get_response(fakes.wsgi_app()) self.assertTrue('test1' not in fakes.FakeAuthManager.projects) @@ -89,7 +89,7 @@ class AccountsTest(test.TestCase): def test_account_create(self): body = dict(account=dict(description='test account', manager='guy1')) - req = webob.Request.blank('/v1.0/newacct') + req = webob.Request.blank('/v1.0/accounts/newacct') req.method = 'PUT' req.body = json.dumps(body) @@ -108,7 +108,7 @@ class AccountsTest(test.TestCase): def test_account_update(self): body = dict(account=dict(description='test account', manager='guy2')) - req = webob.Request.blank('/v1.0/test1') + req = webob.Request.blank('/v1.0/accounts/test1') req.method = 'PUT' req.body = json.dumps(body) diff --git a/nova/tests/api/openstack/test_adminapi.py b/nova/tests/api/openstack/test_adminapi.py index 7cb9e8450..4568cb9f5 100644 --- a/nova/tests/api/openstack/test_adminapi.py +++ b/nova/tests/api/openstack/test_adminapi.py @@ -50,7 +50,7 @@ class AdminAPITest(test.TestCase): def test_admin_enabled(self): FLAGS.allow_admin_api = True # We should still be able to access public operations. - req = webob.Request.blank('/v1.0/testacct/flavors') + req = webob.Request.blank('/v1.0/flavors') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) # TODO: Confirm admin operations are available. @@ -58,7 +58,7 @@ class AdminAPITest(test.TestCase): def test_admin_disabled(self): FLAGS.allow_admin_api = False # We should still be able to access public operations. - req = webob.Request.blank('/v1.0/testacct/flavors') + req = webob.Request.blank('/v1.0/flavors') res = req.get_response(fakes.wsgi_app()) # TODO: Confirm admin operations are unavailable. self.assertEqual(res.status_int, 200) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 8268a6fb9..49f90879d 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -78,7 +78,7 @@ class Test(test.TestCase): self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) self.assertEqual(result.headers['X-Server-Management-Url'], - "http://foo/v1.0/test/") + "http://foo/v1.0/") self.assertEqual(result.headers['X-CDN-Management-Url'], "") self.assertEqual(result.headers['X-Storage-Url'], "") diff --git a/nova/tests/api/openstack/test_flavors.py b/nova/tests/api/openstack/test_flavors.py index ba0785b0e..8280a505f 100644 --- a/nova/tests/api/openstack/test_flavors.py +++ b/nova/tests/api/openstack/test_flavors.py @@ -42,11 +42,11 @@ class FlavorsTest(test.TestCase): super(FlavorsTest, self).tearDown() def test_get_flavor_list(self): - req = webob.Request.blank('/v1.0/testacct/flavors') + req = webob.Request.blank('/v1.0/flavors') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) def test_get_flavor_by_id(self): - req = webob.Request.blank('/v1.0/testacct/flavors/1') + req = webob.Request.blank('/v1.0/flavors/1') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 819ca001e..dbe507f7d 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -216,7 +216,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): super(ImageControllerWithGlanceServiceTest, self).tearDown() def test_get_image_index(self): - req = webob.Request.blank('/v1.0/testacct/images') + req = webob.Request.blank('/v1.0/images') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -228,7 +228,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): "image %s not in fixture index!" % str(image)) def test_get_image_details(self): - req = webob.Request.blank('/v1.0/testacct/images/detail') + req = webob.Request.blank('/v1.0/images/detail') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index d592e06b0..705a2f800 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -150,7 +150,7 @@ class ServersTest(test.TestCase): super(ServersTest, self).tearDown() def test_get_server_by_id(self): - req = webob.Request.blank('/v1.0/testacct/servers/1') + req = webob.Request.blank('/v1.0/servers/1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(res_dict['server']['id'], '1') @@ -161,7 +161,7 @@ class ServersTest(test.TestCase): public = ["1.2.3.4"] new_return_server = return_server_with_addresses(private, public) self.stubs.Set(nova.db.api, 'instance_get', new_return_server) - req = webob.Request.blank('/v1.0/testacct/servers/1') + req = webob.Request.blank('/v1.0/servers/1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(res_dict['server']['id'], '1') @@ -173,7 +173,7 @@ class ServersTest(test.TestCase): self.assertEqual(addresses["private"][0], private) def test_get_server_list(self): - req = webob.Request.blank('/v1.0/testacct/servers') + req = webob.Request.blank('/v1.0/servers') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -224,7 +224,7 @@ class ServersTest(test.TestCase): name='server_test', imageId=2, flavorId=2, metadata={'hello': 'world', 'open': 'stack'}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers') + req = webob.Request.blank('/v1.0/servers') req.method = 'POST' req.body = json.dumps(body) @@ -233,7 +233,7 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) def test_update_no_body(self): - req = webob.Request.blank('/v1.0/testacct/servers/1') + req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 422) @@ -251,7 +251,7 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'instance_update', server_update) - req = webob.Request.blank('/v1.0/testacct/servers/1') + req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' req.body = self.body req.get_response(fakes.wsgi_app()) @@ -267,30 +267,30 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'instance_update', server_update) - req = webob.Request.blank('/v1.0/testacct/servers/1') + req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' req.body = self.body req.get_response(fakes.wsgi_app()) def test_create_backup_schedules(self): - req = webob.Request.blank('/v1.0/testacct/servers/1/backup_schedules') + req = webob.Request.blank('/v1.0/servers/1/backup_schedules') req.method = 'POST' res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_delete_backup_schedules(self): - req = webob.Request.blank('/v1.0/testacct/servers/1/backup_schedules') + req = webob.Request.blank('/v1.0/servers/1/backup_schedules') req.method = 'DELETE' res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_get_server_backup_schedules(self): - req = webob.Request.blank('/v1.0/testacct/servers/1/backup_schedules') + req = webob.Request.blank('/v1.0/servers/1/backup_schedules') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_get_all_server_details(self): - req = webob.Request.blank('/v1.0/testacct/servers/detail') + req = webob.Request.blank('/v1.0/servers/detail') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -321,7 +321,7 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'instance_get_all_by_user', return_servers_with_host) - req = webob.Request.blank('/v1.0/testacct/servers/detail') + req = webob.Request.blank('/v1.0/servers/detail') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -341,7 +341,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/pause') + req = webob.Request.blank('/v1.0/servers/1/pause') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -353,7 +353,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/unpause') + req = webob.Request.blank('/v1.0/servers/1/unpause') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -365,7 +365,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/suspend') + req = webob.Request.blank('/v1.0/servers/1/suspend') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -377,7 +377,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/resume') + req = webob.Request.blank('/v1.0/servers/1/resume') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -389,7 +389,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/reset_network') + req = webob.Request.blank('/v1.0/servers/1/reset_network') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -402,7 +402,7 @@ class ServersTest(test.TestCase): name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) req = webob.Request.blank( - '/v1.0/testacct/servers/1/inject_network_info') + '/v1.0/servers/1/inject_network_info') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -410,13 +410,13 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 202) def test_server_diagnostics(self): - req = webob.Request.blank("/v1.0/testacct/servers/1/diagnostics") + req = webob.Request.blank("/v1.0/servers/1/diagnostics") req.method = "GET" res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 404) def test_server_actions(self): - req = webob.Request.blank("/v1.0/testacct/servers/1/actions") + req = webob.Request.blank("/v1.0/servers/1/actions") req.method = "GET" res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 404) @@ -425,7 +425,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/action') + req = webob.Request.blank('/v1.0/servers/1/action') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -435,7 +435,7 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/action') + req = webob.Request.blank('/v1.0/servers/1/action') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) @@ -445,14 +445,14 @@ class ServersTest(test.TestCase): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, personality={})) - req = webob.Request.blank('/v1.0/testacct/servers/1/action') + req = webob.Request.blank('/v1.0/servers/1/action') req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) res = req.get_response(fakes.wsgi_app()) def test_delete_server_instance(self): - req = webob.Request.blank('/v1.0/testacct/servers/1') + req = webob.Request.blank('/v1.0/servers/1') req.method = 'DELETE' self.server_delete_called = False diff --git a/nova/tests/api/openstack/test_users.py b/nova/tests/api/openstack/test_users.py index bd32254cd..14c7897f0 100644 --- a/nova/tests/api/openstack/test_users.py +++ b/nova/tests/api/openstack/test_users.py @@ -72,7 +72,7 @@ class UsersTest(test.TestCase): super(UsersTest, self).tearDown() def test_get_user_list(self): - req = webob.Request.blank('/v1.0/testacct/users') + req = webob.Request.blank('/v1.0/users') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -80,7 +80,7 @@ class UsersTest(test.TestCase): self.assertEqual(len(res_dict['users']), 2) def test_get_user_by_id(self): - req = webob.Request.blank('/v1.0/testacct/users/guy2') + req = webob.Request.blank('/v1.0/users/guy2') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -91,7 +91,7 @@ class UsersTest(test.TestCase): self.assertEqual(res.status_int, 200) def test_user_delete(self): - req = webob.Request.blank('/v1.0/testacct/users/guy1') + req = webob.Request.blank('/v1.0/users/guy1') req.method = 'DELETE' res = req.get_response(fakes.wsgi_app()) self.assertTrue('guy1' not in [u.id for u in @@ -103,7 +103,7 @@ class UsersTest(test.TestCase): access='acc3', secret='invasionIsInNormandy', admin=True)) - req = webob.Request.blank('/v1.0/testacct/users') + req = webob.Request.blank('/v1.0/users') req.method = 'POST' req.body = json.dumps(body) @@ -124,7 +124,7 @@ class UsersTest(test.TestCase): body = dict(user=dict(name='guy2', access='acc2', secret='invasionIsInNormandy')) - req = webob.Request.blank('/v1.0/testacct/users/guy2') + req = webob.Request.blank('/v1.0/users/guy2') req.method = 'PUT' req.body = json.dumps(body) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 51f13af48..6d869dc15 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -85,7 +85,7 @@ class ZonesTest(test.TestCase): super(ZonesTest, self).tearDown() def test_get_zone_list(self): - req = webob.Request.blank('/v1.0/testacct/zones') + req = webob.Request.blank('/v1.0/zones') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -93,7 +93,7 @@ class ZonesTest(test.TestCase): self.assertEqual(len(res_dict['zones']), 2) def test_get_zone_by_id(self): - req = webob.Request.blank('/v1.0/testacct/zones/1') + req = webob.Request.blank('/v1.0/zones/1') res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) @@ -103,7 +103,7 @@ class ZonesTest(test.TestCase): self.assertEqual(res.status_int, 200) def test_zone_delete(self): - req = webob.Request.blank('/v1.0/testacct/zones/1') + req = webob.Request.blank('/v1.0/zones/1') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) @@ -111,7 +111,7 @@ class ZonesTest(test.TestCase): def test_zone_create(self): body = dict(zone=dict(api_url='http://blah.zoo', username='fred', password='fubar')) - req = webob.Request.blank('/v1.0/testacct/zones') + req = webob.Request.blank('/v1.0/zones') req.method = 'POST' req.body = json.dumps(body) @@ -125,7 +125,7 @@ class ZonesTest(test.TestCase): def test_zone_update(self): body = dict(zone=dict(username='zeb', password='sneaky')) - req = webob.Request.blank('/v1.0/testacct/zones/1') + req = webob.Request.blank('/v1.0/zones/1') req.method = 'PUT' req.body = json.dumps(body) -- cgit From e81294b94e3bc8708bd4777b685a7d302594557e Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Tue, 8 Mar 2011 18:53:20 +0100 Subject: Added ability to remove networks on nova-manage command --- nova/db/api.py | 7 +++++++ nova/db/sqlalchemy/api.py | 7 +++++++ 2 files changed, 14 insertions(+) (limited to 'nova') diff --git a/nova/db/api.py b/nova/db/api.py index 04f5fd72f..7ad99c1f4 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -488,6 +488,13 @@ def network_create_safe(context, values): """ return IMPL.network_create_safe(context, values) +def network_delete_safe(context, network_id): + """Delete network with key network_id + + This method assumes that the network is not associated with any project + """ + return IMPL.network_delete_safe(context, network_id) + def network_create_fixed_ips(context, network_id, num_vpn_clients): """Create the ips for the network, reserving sepecified ips.""" diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index c8f42425d..90730d325 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1042,6 +1042,13 @@ def network_create_safe(context, values): except IntegrityError: return None +@require_admin_context +def network_delete_safe(context, network_id): + session = get_session() + with session.begin(): + network_ref = network_get(context, network_id=network_id, session=session) + session.delete(network_ref) + @require_admin_context def network_disassociate(context, network_id): -- cgit From 23d3be4b6f28359211e29212867157daeac9e142 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 8 Mar 2011 20:25:05 +0000 Subject: update code to work with new container and disk formats from glance --- nova/api/ec2/cloud.py | 9 ++++++--- nova/image/s3.py | 7 ++++++- 2 files changed, 12 insertions(+), 4 deletions(-) (limited to 'nova') diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 6479c9445..6b79f7f53 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -867,7 +867,8 @@ class CloudController(object): def _format_image(self, image): """Convert from format defined by BaseImageService to S3 format.""" i = {} - ec2_id = self._image_ec2_id(image.get('id'), image.get('type')) + image_type = image['properties'].get('type') + ec2_id = self._image_ec2_id(image.get('id'), image_type) name = image.get('name') if name: i['imageId'] = "%s (%s)" % (ec2_id, name) @@ -882,7 +883,7 @@ class CloudController(object): i['imageOwnerId'] = image['properties'].get('owner_id') i['imageLocation'] = image['properties'].get('image_location') i['imageState'] = image['properties'].get('image_state') - i['type'] = image.get('type') + i['type'] = image_type i['isPublic'] = str(image['properties'].get('is_public', '')) == 'True' i['architecture'] = image['properties'].get('architecture') return i @@ -915,7 +916,8 @@ class CloudController(object): image_location = kwargs['name'] metadata = {'properties': {'image_location': image_location}} image = self.image_service.create(context, metadata) - image_id = self._image_ec2_id(image['id'], image['type']) + image_id = self._image_ec2_id(image['id'], + image['properties']['type']) msg = _("Registered image %(image_location)s with" " id %(image_id)s") % locals() LOG.audit(msg, context=context) @@ -954,6 +956,7 @@ class CloudController(object): raise exception.NotFound(_('Image %s not found') % image_id) internal_id = image['id'] del(image['id']) + raise Exception(image) image['properties']['is_public'] = (operation_type == 'add') return self.image_service.update(context, internal_id, image) diff --git a/nova/image/s3.py b/nova/image/s3.py index ab6eea8cf..bf104c29a 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -139,11 +139,13 @@ class S3ImageService(service.BaseImageService): manifest = key.get_contents_as_string() manifest = ElementTree.fromstring(manifest) + image_format = 'ami' image_type = 'machine' try: kernel_id = manifest.find("machine_configuration/kernel_id").text if kernel_id == 'true': + image_format = 'aki' image_type = 'kernel' kernel_id = None except: @@ -152,6 +154,7 @@ class S3ImageService(service.BaseImageService): try: ramdisk_id = manifest.find("machine_configuration/ramdisk_id").text if ramdisk_id == 'true': + image_format = 'ari' image_type = 'ramdisk' ramdisk_id = None except: @@ -173,7 +176,9 @@ class S3ImageService(service.BaseImageService): properties['ramdisk_id'] = ec2utils.ec2_id_to_id(ramdisk_id) properties['is_public'] = False - metadata.update({'type': image_type, + properties['type'] = image_type + metadata.update({'disk_format': image_format, + 'container_format': image_format, 'status': 'queued', 'is_public': True, 'properties': properties}) -- cgit From e4b176d41cca234082c28ba6d9188745f1d2b98a Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Wed, 9 Mar 2011 00:49:56 +0000 Subject: a few fixes for the tests --- .../versions/009_add_os_type_to_instances.py | 53 ---------------------- .../versions/010_add_os_type_to_instances.py | 53 ++++++++++++++++++++++ nova/tests/test_xenapi.py | 1 + nova/virt/xenapi/vmops.py | 2 +- 4 files changed, 55 insertions(+), 54 deletions(-) delete mode 100644 nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py (limited to 'nova') diff --git a/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py deleted file mode 100644 index 514b92b81..000000000 --- a/nova/db/sqlalchemy/migrate_repo/versions/009_add_os_type_to_instances.py +++ /dev/null @@ -1,53 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import * -from sqlalchemy.sql import text -from migrate import * - -from nova import log as logging - - -meta = MetaData() - -instances = Table('instances', meta, - Column('id', Integer(), primary_key=True, nullable=False), - ) - -instances_os_type = Column('os_type', - String(length=255, convert_unicode=False, - assert_unicode=None, unicode_error=None, - _warn_on_bytestring=False), - nullable=True) - - -def upgrade(migrate_engine): - # Upgrade operations go here. Don't create your own engine; - # bind migrate_engine to your metadata - meta.bind = migrate_engine - - instances.create_column(instances_os_type) - migrate_engine.execute(instances.update()\ - .where(instances.c.os_type == None)\ - .values(os_type='linux')) - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - - instances.drop_column('os_type') diff --git a/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py new file mode 100644 index 000000000..514b92b81 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py @@ -0,0 +1,53 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from sqlalchemy.sql import text +from migrate import * + +from nova import log as logging + + +meta = MetaData() + +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +instances_os_type = Column('os_type', + String(length=255, convert_unicode=False, + assert_unicode=None, unicode_error=None, + _warn_on_bytestring=False), + nullable=True) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + instances.create_column(instances_os_type) + migrate_engine.execute(instances.update()\ + .where(instances.c.os_type == None)\ + .values(os_type='linux')) + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + + instances.drop_column('os_type') diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 383819b00..cd125a301 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -440,6 +440,7 @@ class XenAPIMigrateInstance(test.TestCase): 'ramdisk_id': None, 'instance_type': 'm1.large', 'mac_address': 'aa:bb:cc:dd:ee:ff', + 'os_type': 'linux' } stubs.stub_out_migration_methods(self.stubs) glance_stubs.stubout_glance_client(self.stubs, diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index a6a9fbf95..aa4372c3d 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -261,7 +261,7 @@ class VMOps(object): template_vm_ref, template_vdi_uuids = self._get_snapshot(instance) # call plugin to ship snapshot off to glance VMHelper.upload_image( - self._session, instance.id, template_vdi_uuids, image_id) + self._session, instance, template_vdi_uuids, image_id) finally: if template_vm_ref: self._destroy(instance, template_vm_ref, -- cgit From a320b5df9f916adf8422ed312306c77570d392c2 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 00:30:05 -0500 Subject: execvp: almost passes tests --- nova/api/ec2/cloud.py | 2 +- nova/network/linux_net.py | 21 +++++++++++---------- nova/tests/test_network.py | 2 +- nova/tests/test_virt.py | 11 ++++++----- nova/utils.py | 19 +++++++++++++------ nova/virt/libvirt_conn.py | 11 ++++++----- nova/virt/xenapi/vm_utils.py | 6 ++---- nova/volume/driver.py | 5 +++-- 8 files changed, 43 insertions(+), 34 deletions(-) (limited to 'nova') diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 0d22a3f46..b7d72d1c1 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -115,7 +115,7 @@ class CloudController(object): start = os.getcwd() os.chdir(FLAGS.ca_path) # TODO(vish): Do this with M2Crypto instead - utils.runthis(_("Generating root CA: %s"), "sh genrootca.sh") + utils.runthis(_("Generating root CA: %s"), "sh", "genrootca.sh") os.chdir(start) def _get_mpi_data(self, context, project_id): diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index ad019a8c0..b66a1adb7 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -139,14 +139,14 @@ def init_host(): def bind_floating_ip(floating_ip, check_exit_code=True): """Bind ip to public interface""" _execute('sudo', 'ip', 'addr', 'add', floating_ip, - 'dev', FLAGS.public_interface), + 'dev', FLAGS.public_interface, check_exit_code=check_exit_code) def unbind_floating_ip(floating_ip): """Unbind a public ip from public interface""" _execute('sudo', 'ip', 'addr', 'del', floating_ip, - 'dev', FLAGS.public_interface)) + 'dev', FLAGS.public_interface) def ensure_vlan_forward(public_ip, port, private_ip): @@ -213,7 +213,7 @@ def ensure_bridge(bridge, interface, net_attrs=None): _execute('sudo', 'brctl', 'addbr', bridge) _execute('sudo', 'brctl', 'setfd', bridge, 0) # _execute("sudo brctl setageing %s 10" % bridge) - _execute('sudo', 'brctl', 'stp', bridge', 'off') + _execute('sudo', 'brctl', 'stp', bridge, 'off') _execute('sudo', 'ip', 'link', 'set', bridge, up) if net_attrs: # NOTE(vish): The ip for dnsmasq has to be the first address on the @@ -223,7 +223,7 @@ def ensure_bridge(bridge, interface, net_attrs=None): "%s/%s" % (net_attrs['gateway'], suffix), 'brd', - net-attrs['broadcast'], + net_attrs['broadcast'], 'dev', bridge, check_exit_code=False) @@ -257,7 +257,7 @@ def ensure_bridge(bridge, interface, net_attrs=None): _execute('sudo', 'ip', 'addr', 'add', params, 'dev', bridge) if gateway: _execute('sudo', 'route', 'add', '0.0.0.0', 'gw', gateway) - out, err = _execute('sudo', 'brctl', 'addif, bridge, interface, + out, err = _execute('sudo', 'brctl', 'addif', bridge, interface, check_exit_code=False) if (err and err != "device %s is already a member of a bridge; can't " @@ -265,11 +265,11 @@ def ensure_bridge(bridge, interface, net_attrs=None): raise exception.Error("Failed to add interface: %s" % err) if FLAGS.use_nova_chains: - (out, err) = _execute('sudo', 'iptables', '-N', 'nova_forward, + (out, err) = _execute('sudo', 'iptables', '-N', 'nova_forward', check_exit_code=False) if err != 'iptables: Chain already exists.\n': # NOTE(vish): chain didn't exist link chain - _execute('sudo', 'iptables, '-D', 'FORWARD', '-j', 'nova_forward', + _execute('sudo', 'iptables', '-D', 'FORWARD', '-j', 'nova_forward', check_exit_code=False) _execute('sudo', 'iptables', '-A', 'FORWARD', '-j', 'nova_forward') @@ -355,7 +355,7 @@ interface %s # if radvd is already running, then tell it to reload if pid: - out, _err = _execute('cat', "/proc/%d/cmdline' + out, _err = _execute('cat', '/proc/%d/cmdline' % pid, check_exit_code=False) if conffile in out: try: @@ -383,7 +383,7 @@ def _host_dhcp(fixed_ip_ref): def _execute(*cmd, **kwargs): """Wrapper around utils._execute for fake_network""" if FLAGS.fake_network: - LOG.debug("FAKE NET: %s", ' '.join(cmd)) + LOG.debug("FAKE NET: %s", " ".join(map(str, cmd))) return "fake", 0 else: return utils.execute(*cmd, **kwargs) @@ -396,7 +396,8 @@ def _device_exists(device): return not err -def _confirm_rule(chain, *cmd, append=False): +def _confirm_rule(chain, *cmd, **kwargs): + append=kwargs.get('append',False) """Delete and re-add iptables rule""" if FLAGS.use_nova_chains: chain = "nova_%s" % chain.lower() diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 6d2d8b771..19099ff4c 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -343,7 +343,7 @@ def lease_ip(private_ip): private_ip) instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) - cmd = (binpath('nova-dhcpbridge'), 'add' + cmd = (binpath('nova-dhcpbridge'), 'add', instance_ref['mac_address'], private_ip, 'fake') env = {'DNSMASQ_INTERFACE': network_ref['bridge'], diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index f151ae911..7f1ad002e 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -315,15 +315,16 @@ class IptablesFirewallTestCase(test.TestCase): instance_ref = db.instance_get(admin_ctxt, instance_ref['id']) # self.fw.add_instance(instance_ref) - def fake_iptables_execute(cmd, process_input=None): - if cmd == 'sudo ip6tables-save -t filter': + def fake_iptables_execute(*cmd, **kwargs): + process_input=kwargs.get('process_input', None) + if cmd == ('sudo', 'ip6tables-save', '-t', 'filter'): return '\n'.join(self.in6_rules), None - if cmd == 'sudo iptables-save -t filter': + if cmd == ('sudo', 'iptables-save', '-t', 'filter'): return '\n'.join(self.in_rules), None - if cmd == 'sudo iptables-restore': + if cmd == ('sudo', 'iptables-restore'): self.out_rules = process_input.split('\n') return '', '' - if cmd == 'sudo ip6tables-restore': + if cmd == ('sudo', 'ip6tables-restore'): self.out6_rules = process_input.split('\n') return '', '' self.fw.execute = fake_iptables_execute diff --git a/nova/utils.py b/nova/utils.py index c96b85294..9b51f8b40 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -128,13 +128,20 @@ def fetchfile(url, target): execute("curl", "--fail", url, "-o", target) -def execute(*cmd, process_input=None, addl_env=None, check_exit_code=True): +def execute(*cmd, **kwargs): + process_input=kwargs.get('process_input', None) + addl_env=kwargs.get('addl_env', None) + check_exit_code=kwargs.get('check_exit_code', True) + stdin=kwargs.get('stdin', subprocess.PIPE) + stdout=kwargs.get('stdout', subprocess.PIPE) + stderr=kwargs.get('stderr', subprocess.PIPE) + LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(*cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + obj = subprocess.Popen(cmd, stdin=stdin, + stdout=stdout, stderr=stderr, env=env) result = None if process_input != None: result = obj.communicate(process_input) @@ -220,9 +227,9 @@ def debug(arg): return arg -def runthis(prompt, cmd, check_exit_code=True): - LOG.debug(_("Running %s"), (cmd)) - rv, err = execute(cmd, check_exit_code=check_exit_code) +def runthis(prompt, *cmd, **kwargs): + LOG.debug(_("Running %s"), (" ".join(cmd))) + rv, err = execute(*cmd, **kwargs) def generate_uid(topic, size=8): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index a5256bbc2..76f31f91a 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -540,8 +540,8 @@ class LibvirtConnection(object): if not os.path.exists(base): fn(target=base, *args, **kwargs) if cow: - utils.execute('qemu-img', 'create', '-f', 'qcow2', "'-o'', - "cluster_size=2M,backing_file=%s" % base, + utils.execute('qemu-img', 'create', '-f', 'qcow2', '-o', + 'cluster_size=2M,backing_file=%s' % base, target) else: utils.execute('cp', base, target) @@ -554,7 +554,7 @@ class LibvirtConnection(object): def _create_local(self, target, local_gb): """Create a blank image of specified size""" - utils.execute('truncate', target, '-s', "%dG" local_gb) + utils.execute('truncate', target, '-s', "%dG" % local_gb) # TODO(vish): should we format disk by default? def _create_image(self, inst, libvirt_xml, suffix='', disk_images=None): @@ -565,7 +565,7 @@ class LibvirtConnection(object): fname + suffix) # ensure directories exist and are writable - utils.execute('mkdir', '-p', basepath(suffix='') + utils.execute('mkdir', '-p', basepath(suffix='')) LOG.info(_('instance %s: Creating image'), inst['name']) f = open(basepath('libvirt.xml'), 'w') @@ -1245,7 +1245,8 @@ class IptablesFirewallDriver(FirewallDriver): self.apply_ruleset() def apply_ruleset(self): - current_filter, _ = self.execute('sudo iptables-save -t filter') + current_filter, _ = self.execute('sudo', 'iptables-save', + '-t', 'filter') current_lines = current_filter.split('\n') new_filter = self.modify_rules(current_lines, 4) self.execute('sudo', 'iptables-restore', diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 8fdb658fb..6ba13f980 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -915,10 +915,8 @@ def _write_partition(virtual_size, dev): LOG.debug(_('Writing partition table %(primary_first)d %(primary_last)d' ' to %(dest)s...') % locals()) - def execute(*cmd, process_input=None, check_exit_code=True): - return utils.execute(*cmd, - process_input=process_input, - check_exit_code=check_exit_code) + def execute(*cmd, **kwargs): + return utils.execute(*cmd, **kwargs) execute('parted', '--script', dest, 'mklabel', 'msdos') execute('parted', '--script', dest, 'mkpart', 'primary', diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 220c9ef9d..e9bdf162f 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -104,7 +104,8 @@ class VolumeDriver(object): def delete_volume(self, volume): """Deletes a logical volume.""" try: - self._try_execute('sudo', 'lvdisplay', '%s/%s" % + self._try_execute('sudo', 'lvdisplay', + '%s/%s' % (FLAGS.volume_group, volume['name'])) except Exception as e: @@ -550,7 +551,7 @@ class SheepdogDriver(VolumeDriver): else: sizestr = '%sG' % volume['size'] self._try_execute('qemu-img', 'create', - "sheepdog:%s" %s" % volume['name'], + "sheepdog:%s" % volume['name'], sizestr) def delete_volume(self, volume): -- cgit From 1d7358e70379607c9cce02307f4336efbd135a5d Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 01:26:53 -0500 Subject: execvp: unit tests pass --- nova/crypto.py | 2 +- nova/utils.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/crypto.py b/nova/crypto.py index dd24723b8..20bb570a5 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -196,7 +196,7 @@ def generate_x509_cert(user_id, project_id, bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key')) csrfile = os.path.join(tmpdir, 'temp.csr') - utils.execute('openssl', 'genrsa', '-out', keyfile, bits) + utils.execute('openssl', 'genrsa', '-out', keyfile, str(bits)) utils.execute('openssl', 'req', '-new', '-key', keyfile, '-out', csrfile, '-batch', '-subj', subject) private_key = open(keyfile).read() diff --git a/nova/utils.py b/nova/utils.py index 9b51f8b40..7ddf056ea 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -135,6 +135,7 @@ def execute(*cmd, **kwargs): stdin=kwargs.get('stdin', subprocess.PIPE) stdout=kwargs.get('stdout', subprocess.PIPE) stderr=kwargs.get('stderr', subprocess.PIPE) + cmd=map(str,cmd) LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() -- cgit From 77da93886be61230dea5a4a4c4de036a57e62550 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 9 Mar 2011 06:56:42 +0000 Subject: tests and semaphore fix for image caching --- nova/tests/test_virt.py | 66 +++++++++++++++++++++++++++++++++++++++++++++++ nova/virt/libvirt_conn.py | 14 +++++++--- 2 files changed, 76 insertions(+), 4 deletions(-) (limited to 'nova') diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index f151ae911..ec7498d72 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -14,12 +14,16 @@ # License for the specific language governing permissions and limitations # under the License. +import os + +import eventlet from xml.etree.ElementTree import fromstring as xml_to_tree from xml.dom.minidom import parseString as xml_to_dom from nova import context from nova import db from nova import flags +from nova import log as logging from nova import test from nova import utils from nova.api.ec2 import cloud @@ -30,6 +34,68 @@ FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') +def _concurrency(wait, done, target): + wait.wait() + done.send() + + +class CacheConcurrencyTestCase(test.TestCase): + def setUp(self): + super(CacheConcurrencyTestCase, self).setUp() + + def fake_exists(fname): + basedir = os.path.join(FLAGS.instances_path, '_base') + if fname == basedir: + return True + return False + + def fake_execute(*args, **kwargs): + pass + + self.stubs.Set(os.path, 'exists', fake_exists) + self.stubs.Set(utils, 'execute', fake_execute) + + def test_same_fname_concurrency(self): + """Ensures that the same fname cache runs at a sequentially""" + conn = libvirt_conn.get_connection(False) + wait1 = eventlet.event.Event() + done1 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname', False, wait1, done1) + wait2 = eventlet.event.Event() + done2 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname', False, wait2, done2) + wait2.send() + eventlet.sleep(0) + try: + self.assertFalse(done2.ready()) + finally: + wait1.send() + done1.wait() + eventlet.sleep(0) + self.assertTrue(done2.ready()) + + def test_different_fname_concurrency(self): + """Ensures that two different fname caches are concurrent""" + conn = libvirt_conn.get_connection(False) + wait1 = eventlet.event.Event() + done1 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname2', False, wait1, done1) + wait2 = eventlet.event.Event() + done2 = eventlet.event.Event() + eventlet.spawn(conn._cache_image, _concurrency, + 'target', 'fname1', False, wait2, done2) + wait2.send() + eventlet.sleep(0) + try: + self.assertTrue(done2.ready()) + finally: + wait1.send() + eventlet.sleep(0) + + class LibvirtConnTestCase(test.TestCase): def setUp(self): super(LibvirtConnTestCase, self).setUp() diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 9f7315c17..1a1f146d4 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -44,9 +44,8 @@ import uuid from xml.dom import minidom -from eventlet import greenthread -from eventlet import event from eventlet import tpool +from eventlet import semaphore import IPy @@ -512,6 +511,8 @@ class LibvirtConnection(object): subprocess.Popen(cmd, shell=True) return {'token': token, 'host': host, 'port': port} + _image_semaphores = {} + def _cache_image(self, fn, target, fname, cow=False, *args, **kwargs): """Wrapper for a method that creates an image that caches the image. @@ -531,8 +532,13 @@ class LibvirtConnection(object): if not os.path.exists(base_dir): os.mkdir(base_dir) base = os.path.join(base_dir, fname) - if not os.path.exists(base): - fn(target=base, *args, **kwargs) + + if fname not in self._image_semaphores: + self._image_semaphores[fname] = semaphore.Semaphore() + with self._image_semaphores[fname]: + if not os.path.exists(base): + fn(target=base, *args, **kwargs) + if cow: utils.execute('qemu-img create -f qcow2 -o ' 'cluster_size=2M,backing_file=%s %s' -- cgit From ddeab2da30bb2f74109854d982c6681e78e7a4ce Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 9 Mar 2011 07:35:58 +0000 Subject: make static method for testing without initializing libvirt --- nova/tests/test_virt.py | 4 ++-- nova/virt/libvirt_conn.py | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) (limited to 'nova') diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index ec7498d72..113632a0c 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -57,7 +57,7 @@ class CacheConcurrencyTestCase(test.TestCase): def test_same_fname_concurrency(self): """Ensures that the same fname cache runs at a sequentially""" - conn = libvirt_conn.get_connection(False) + conn = libvirt_conn.LibvirtConnection wait1 = eventlet.event.Event() done1 = eventlet.event.Event() eventlet.spawn(conn._cache_image, _concurrency, @@ -78,7 +78,7 @@ class CacheConcurrencyTestCase(test.TestCase): def test_different_fname_concurrency(self): """Ensures that two different fname caches are concurrent""" - conn = libvirt_conn.get_connection(False) + conn = libvirt_conn.LibvirtConnection wait1 = eventlet.event.Event() done1 = eventlet.event.Event() eventlet.spawn(conn._cache_image, _concurrency, diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 1a1f146d4..ecef7950a 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -511,9 +511,10 @@ class LibvirtConnection(object): subprocess.Popen(cmd, shell=True) return {'token': token, 'host': host, 'port': port} - _image_semaphores = {} + _image_sems = {} - def _cache_image(self, fn, target, fname, cow=False, *args, **kwargs): + @staticmethod + def _cache_image(fn, target, fname, cow=False, *args, **kwargs): """Wrapper for a method that creates an image that caches the image. This wrapper will save the image into a common store and create a @@ -533,9 +534,9 @@ class LibvirtConnection(object): os.mkdir(base_dir) base = os.path.join(base_dir, fname) - if fname not in self._image_semaphores: - self._image_semaphores[fname] = semaphore.Semaphore() - with self._image_semaphores[fname]: + if fname not in LibvirtConnection._image_sems: + LibvirtConnection._image_sems[fname] = semaphore.Semaphore() + with LibvirtConnection._image_sems[fname]: if not os.path.exists(base): fn(target=base, *args, **kwargs) -- cgit From 7d31fe9ef316f49379818259a55a84deb5b850cd Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 9 Mar 2011 10:30:18 +0100 Subject: Stop assuming anything about the order in which the two processes are scheduled. --- nova/tests/test_misc.py | 47 +++++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 22 deletions(-) (limited to 'nova') diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index 9f572b58e..a658e4978 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -71,30 +71,33 @@ class LockTestCase(test.TestCase): "got mangled") def test_synchronized(self): - rpipe, wpipe = os.pipe() + rpipe1, wpipe1 = os.pipe() + rpipe2, wpipe2 = os.pipe() + + @synchronized('testlock') + def f(rpipe, wpipe): + try: + os.write(wpipe, "foo") + except OSError, e: + self.assertEquals(e.errno, errno.EPIPE) + return + + rfds, _, __ = select.select([rpipe], [], [], 1) + self.assertEquals(len(rfds), 0, "The other process, which was" + " supposed to be locked, " + "wrote on its end of the " + "pipe") + os.close(rpipe) + pid = os.fork() if pid > 0: - os.close(wpipe) - - @synchronized('testlock') - def f(): - rfds, _, __ = select.select([rpipe], [], [], 1) - self.assertEquals(len(rfds), 0, "The other process, which was" - " supposed to be locked, " - "wrote on its end of the " - "pipe") - os.close(rpipe) - - f() + os.close(wpipe1) + os.close(rpipe2) + + f(rpipe1, wpipe2) else: - os.close(rpipe) + os.close(rpipe1) + os.close(wpipe2) - @synchronized('testlock') - def g(): - try: - os.write(wpipe, "foo") - except OSError, e: - self.assertEquals(e.errno, errno.EPIPE) - return - g() + f(rpipe2, wpipe1) os._exit(0) -- cgit From 0f45b59ca6f9502a3ae6578e2fca5a7d9575ae5e Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Wed, 9 Mar 2011 10:37:21 -0500 Subject: Added 'adminPass' to the serialization_metadata. --- nova/api/openstack/servers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 9581b8477..bbedd7c63 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -96,7 +96,7 @@ class Controller(wsgi.Controller): 'application/xml': { "attributes": { "server": ["id", "imageId", "name", "flavorId", "hostId", - "status", "progress"]}}} + "status", "progress", "adminPass"]}}} def __init__(self): self.compute_api = compute.API() -- cgit From f1dea606a64c9144fb723be0e5b86806891380f8 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 9 Mar 2011 11:54:21 -0500 Subject: add override to handle xml deserialization for server instance creation --- nova/api/openstack/servers.py | 68 +++++++- nova/tests/api/openstack/fakes.py | 2 - nova/tests/api/openstack/test_servers.py | 277 +++++++++++++++++++++++++++++++ 3 files changed, 344 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 93f504f91..4a6282204 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -17,6 +17,7 @@ import base64 import hashlib import json import traceback +from xml.dom import minidom from webob import exc @@ -166,9 +167,16 @@ class Controller(wsgi.Controller): personality_files.append((path, contents)) return personality_files + def _deserialize_create(self, request): + if request.content_type == "application/xml": + deserializer = ServerCreateRequestXMLDeserializer() + return deserializer.deserialize(request.body) + else: + return self._deserialize(request.body, request) + def create(self, req): """ Creates a new server for a given user """ - env = self._deserialize(req.body, req) + env = self._deserialize_create(req) if not env: return faults.Fault(exc.HTTPUnprocessableEntity()) @@ -448,3 +456,61 @@ class Controller(wsgi.Controller): _("Ramdisk not found for image %(image_id)s") % locals()) return kernel_id, ramdisk_id + + +class ServerCreateRequestXMLDeserializer(object): + + def deserialize(self, string): + dom = minidom.parseString(string) + server = self._extract_server(dom) + return {'server': server} + + def _extract_server(self, node): + server = {} + server_node = self._find_first_child_named(node, 'server') + for attr in ["name", "imageId", "flavorId"]: + server[attr] = server_node.getAttribute(attr) + metadata = self._extract_metadata(server_node) + if metadata is not None: + server["metadata"] = metadata + personality = self._extract_personality(server_node) + if personality is not None: + server["personality"] = personality + return server + + def _extract_metadata(self, server_node): + metadata_node = self._find_first_child_named(server_node, "metadata") + if metadata_node is None: + return None + metadata = {} + for meta_node in metadata_node.childNodes: + key = meta_node.getAttribute("key") + metadata[key] = self._extract_text(meta_node) + return metadata + + def _extract_personality(self, server_node): + personality_node = \ + self._find_first_child_named(server_node, "personality") + if personality_node is None: + return None + personality = [] + for file_node in personality_node.childNodes: + item = {} + if file_node.hasAttribute("path"): + item["path"] = file_node.getAttribute("path") + item["contents"] = self._extract_text(file_node) + personality.append(item) + return personality + + def _find_first_child_named(self, parent, name): + for node in parent.childNodes: + if node.nodeName == name: + return node + return None + + def _extract_text(self, node): + if len(node.childNodes) == 1: + child = node.childNodes[0] + if child.nodeType == child.TEXT_NODE: + return child.nodeValue + return "" diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 49ce8c1b5..1d8692528 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -68,8 +68,6 @@ def fake_auth_init(self, application): @webob.dec.wsgify def fake_wsgi(self, req): req.environ['nova.context'] = context.RequestContext(1, 1) - if req.body: - req.environ['inst_dict'] = json.loads(req.body) return self.application diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 8fb5a9aec..5132d1ad3 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -18,6 +18,7 @@ import base64 import datetime import json +import unittest import stubout import webob @@ -284,6 +285,37 @@ class ServersTest(test.TestCase): response = request.get_response(fakes.wsgi_app()) return compute_api, response + def _format_xml_request_body(self, body_dict): + server = body_dict['server'] + body_parts = [] + body_parts.extend([ + '', + '' % ( + server['name'], server['imageId'], server['flavorId'])]) + if 'metadata' in server: + metadata = server['metadata'] + body_parts.append('') + for item in metadata.iteritems(): + body_parts.append('%s' % item) + body_parts.append('') + if 'personality' in server: + personalities = server['personality'] + body_parts.append('') + for file in personalities: + item = (file['path'], file['contents']) + body_parts.append('%s' % item) + body_parts.append('') + body_parts.append('') + return ''.join(body_parts) + + def _get_create_request_xml(self, body_dict): + req = webob.Request.blank('/v1.0/servers') + req.content_type = 'application/xml' + req.method = 'POST' + req.body = self._format_xml_request_body(body_dict) + return req + def _create_instance_with_personality_json(self, personality): body_dict = self._create_personality_request_dict(personality) request = self._get_create_request_json(body_dict) @@ -291,12 +323,25 @@ class ServersTest(test.TestCase): self._run_create_instance_with_mock_compute_api(request) return request, response, compute_api.personality_files + def _create_instance_with_personality_xml(self, personality): + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_xml(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + return request, response, compute_api.personality_files + def test_create_instance_with_no_personality(self): request, response, personality_files = \ self._create_instance_with_personality_json(personality=None) self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, []) + def test_create_instance_with_no_personality_xml(self): + request, response, personality_files = \ + self._create_instance_with_personality_xml(personality=None) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, []) + def test_create_instance_with_personality(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Hello, World!"\n' @@ -307,6 +352,16 @@ class ServersTest(test.TestCase): self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, [(path, contents)]) + def test_create_instance_with_personality_xml(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Hello, World!"\n' + b64contents = base64.b64encode(contents) + personality = [(path, b64contents)] + request, response, personality_files = \ + self._create_instance_with_personality_xml(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, [(path, contents)]) + def test_create_instance_with_personality_no_path(self): personality = [('/remove/this/path', base64.b64encode('my\n\file\ncontents'))] @@ -318,6 +373,17 @@ class ServersTest(test.TestCase): self.assertEquals(response.status_int, 400) self.assertEquals(compute_api.personality_files, None) + def _test_create_instance_with_personality_no_path_xml(self): + personality = [('/remove/this/path', + base64.b64encode('my\n\file\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_xml(body_dict) + request.body = request.body.replace(' path="/remove/this/path"', '') + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + def test_create_instance_with_personality_no_contents(self): personality = [('/test/path', base64.b64encode('remove\nthese\ncontents'))] @@ -605,3 +671,214 @@ class ServersTest(test.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '202 Accepted') self.assertEqual(self.server_delete_called, True) + + +class TestServerCreateRequestXMLDeserializer(unittest.TestCase): + + def setUp(self): + self.deserializer = servers.ServerCreateRequestXMLDeserializer() + + def test_minimal_request(self): + serial_request = """ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + }} + self.assertEquals(request, expected) + + def test_request_with_empty_metadata(self): + serial_request = """ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "metadata": {}, + }} + self.assertEquals(request, expected) + + def test_request_with_empty_personality(self): + serial_request = """ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "personality": [], + }} + self.assertEquals(request, expected) + + def test_request_with_empty_metadata_and_personality(self): + serial_request = """ +\ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "metadata": {}, + "personality": [], + }} + self.assertEquals(request, expected) + + def test_request_with_empty_metadata_and_personality_reversed(self): + serial_request = """ +\ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "metadata": {}, + "personality": [], + }} + self.assertEquals(request, expected) + + def test_request_with_one_personality(self): + serial_request = """ +\ +aabbccdd""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": "aabbccdd"}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_two_personalities(self): + serial_request = """ +\ +aabbccdd\ +abcd""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": "aabbccdd"}, + {"path": "/etc/sudoers", "contents": "abcd"}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_one_personality_missing_path(self): + serial_request = """ +\ +aabbccdd""" + request = self.deserializer.deserialize(serial_request) + expected = [{"contents": "aabbccdd"}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_one_personality_empty_contents(self): + serial_request = """ +\ +""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": ""}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_one_personality_empty_contents_variation(self): + serial_request = """ +\ +""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": ""}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_one_metadata(self): + serial_request = """ +\ +beta""" + request = self.deserializer.deserialize(serial_request) + expected = {"alpha": "beta"} + self.assertEquals(request["server"]["metadata"], expected) + + def test_request_with_two_metadata(self): + serial_request = """ +\ +betabar\ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"alpha": "beta", "foo": "bar"} + self.assertEquals(request["server"]["metadata"], expected) + + def test_request_with_metadata_missing_value(self): + serial_request = """ +\ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"alpha": ""} + self.assertEquals(request["server"]["metadata"], expected) + + def test_request_with_metadata_missing_key(self): + serial_request = """ +\ +beta""" + request = self.deserializer.deserialize(serial_request) + expected = {"": "beta"} + self.assertEquals(request["server"]["metadata"], expected) + + def test_request_with_metadata_duplicate_key(self): + serial_request = """ +\ +barbaz\ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"foo": "baz"} + self.assertEquals(request["server"]["metadata"], expected) + + def test_canonical_request_from_docs(self): + serial_request = """ +\ +Apache1\ +\ +ICAgICAgDQoiQSBjbG91ZCBkb2VzIG5vdCBrbm93IHdoeSBp\ +dCBtb3ZlcyBpbiBqdXN0IHN1Y2ggYSBkaXJlY3Rpb24gYW5k\ +IGF0IHN1Y2ggYSBzcGVlZC4uLkl0IGZlZWxzIGFuIGltcHVs\ +c2lvbi4uLnRoaXMgaXMgdGhlIHBsYWNlIHRvIGdvIG5vdy4g\ +QnV0IHRoZSBza3kga25vd3MgdGhlIHJlYXNvbnMgYW5kIHRo\ +ZSBwYXR0ZXJucyBiZWhpbmQgYWxsIGNsb3VkcywgYW5kIHlv\ +dSB3aWxsIGtub3csIHRvbywgd2hlbiB5b3UgbGlmdCB5b3Vy\ +c2VsZiBoaWdoIGVub3VnaCB0byBzZWUgYmV5b25kIGhvcml6\ +b25zLiINCg0KLVJpY2hhcmQgQmFjaA==\ +""" + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "metadata": { + "My Server Name": "Apache1", + }, + "personality": [ + { + "path": "/etc/banner.txt", + "contents": """\ +ICAgICAgDQoiQSBjbG91ZCBkb2VzIG5vdCBrbm93IHdoeSBp\ +dCBtb3ZlcyBpbiBqdXN0IHN1Y2ggYSBkaXJlY3Rpb24gYW5k\ +IGF0IHN1Y2ggYSBzcGVlZC4uLkl0IGZlZWxzIGFuIGltcHVs\ +c2lvbi4uLnRoaXMgaXMgdGhlIHBsYWNlIHRvIGdvIG5vdy4g\ +QnV0IHRoZSBza3kga25vd3MgdGhlIHJlYXNvbnMgYW5kIHRo\ +ZSBwYXR0ZXJucyBiZWhpbmQgYWxsIGNsb3VkcywgYW5kIHlv\ +dSB3aWxsIGtub3csIHRvbywgd2hlbiB5b3UgbGlmdCB5b3Vy\ +c2VsZiBoaWdoIGVub3VnaCB0byBzZWUgYmV5b25kIGhvcml6\ +b25zLiINCg0KLVJpY2hhcmQgQmFjaA==""", + } + ], + }} + request = self.deserializer.deserialize(serial_request) + self.assertEqual(request, expected) + -- cgit From 1d74816bf705cb672d9d323398b03142297d8bec Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 9 Mar 2011 12:03:17 -0500 Subject: remove superfluous trailing blank line --- nova/tests/api/openstack/test_servers.py | 1 - 1 file changed, 1 deletion(-) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 5132d1ad3..51fb09b18 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -881,4 +881,3 @@ b25zLiINCg0KLVJpY2hhcmQgQmFjaA==""", }} request = self.deserializer.deserialize(serial_request) self.assertEqual(request, expected) - -- cgit From eadce208c55513ddbab550898e641b8ee55a67ec Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Wed, 9 Mar 2011 12:32:15 -0500 Subject: Fix spacing. --- nova/api/openstack/servers.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index bbedd7c63..7222285e0 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -84,11 +84,13 @@ def _translate_detail_keys(inst): return dict(server=inst_dict) + def _translate_keys(inst): """ Coerces into dictionary format, excluding all model attributes save for id and name """ return dict(server=dict(id=inst['id'], name=inst['display_name'])) + class Controller(wsgi.Controller): """ The Server API controller for the OpenStack API """ -- cgit From e17876ec002f976572b6ac102dc113024669a45c Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Wed, 9 Mar 2011 18:57:53 +0100 Subject: fixed lp715427 --- nova/network/manager.py | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'nova') diff --git a/nova/network/manager.py b/nova/network/manager.py index b36dd59cf..39da031eb 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -563,6 +563,16 @@ class VlanManager(NetworkManager): # NOTE(vish): This makes ports unique accross the cloud, a more # robust solution would be to make them unique per ip net['vpn_public_port'] = vpn_start + index + network_ref = None + try: + network_ref = db.network_get_by_cidr(context, cidr) + except exception.NotFound: + pass + + if network_ref is not None: + raise ValueError(_('Network with cidr %s already exists' % + cidr)) + network_ref = self.db.network_create_safe(context, net) if network_ref: self._create_fixed_ips(context, network_ref['id']) -- cgit From 429fdb1ee733a62052c67f4e42c62447fc716ec0 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Wed, 9 Mar 2011 18:10:45 +0000 Subject: removed uneeded **kw args leftover from removed account-in-url changes. --- nova/api/openstack/backup_schedules.py | 6 ++--- nova/api/openstack/consoles.py | 10 ++++---- nova/api/openstack/flavors.py | 6 ++--- nova/api/openstack/images.py | 12 +++++----- nova/api/openstack/servers.py | 42 +++++++++++++++++----------------- nova/api/openstack/shared_ip_groups.py | 12 +++++----- nova/api/openstack/users.py | 12 +++++----- nova/api/openstack/zones.py | 12 +++++----- 8 files changed, 56 insertions(+), 56 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/backup_schedules.py b/nova/api/openstack/backup_schedules.py index a4d5939df..7abb5f884 100644 --- a/nova/api/openstack/backup_schedules.py +++ b/nova/api/openstack/backup_schedules.py @@ -40,15 +40,15 @@ class Controller(wsgi.Controller): def __init__(self): pass - def index(self, req, server_id, **kw): + def index(self, req, server_id): """ Returns the list of backup schedules for a given instance """ return _translate_keys({}) - def create(self, req, server_id, **kw): + def create(self, req, server_id): """ No actual update method required, since the existing API allows both create and update through a POST """ return faults.Fault(exc.HTTPNotImplemented()) - def delete(self, req, server_id, id, **kw): + def delete(self, req, server_id, id): """ Deletes an existing backup schedule """ return faults.Fault(exc.HTTPNotImplemented()) diff --git a/nova/api/openstack/consoles.py b/nova/api/openstack/consoles.py index 85b2a4140..9ebdbe710 100644 --- a/nova/api/openstack/consoles.py +++ b/nova/api/openstack/consoles.py @@ -55,7 +55,7 @@ class Controller(wsgi.Controller): self.console_api = console.API() super(Controller, self).__init__() - def index(self, req, server_id, **kw): + def index(self, req, server_id): """Returns a list of consoles for this instance""" consoles = self.console_api.get_consoles( req.environ['nova.context'], @@ -63,14 +63,14 @@ class Controller(wsgi.Controller): return dict(consoles=[_translate_keys(console) for console in consoles]) - def create(self, req, server_id, **kw): + def create(self, req, server_id): """Creates a new console""" #info = self._deserialize(req.body, req) self.console_api.create_console( req.environ['nova.context'], int(server_id)) - def show(self, req, server_id, id, **kw): + def show(self, req, server_id, id): """Shows in-depth information on a specific console""" try: console = self.console_api.get_console( @@ -81,11 +81,11 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return _translate_detail_keys(console) - def update(self, req, server_id, id, **kw): + def update(self, req, server_id, id): """You can't update a console""" raise faults.Fault(exc.HTTPNotImplemented()) - def delete(self, req, server_id, id, **kw): + def delete(self, req, server_id, id): """Deletes a console""" try: self.console_api.delete_console(req.environ['nova.context'], diff --git a/nova/api/openstack/flavors.py b/nova/api/openstack/flavors.py index 1de67328b..f3d040ba3 100644 --- a/nova/api/openstack/flavors.py +++ b/nova/api/openstack/flavors.py @@ -34,17 +34,17 @@ class Controller(wsgi.Controller): "attributes": { "flavor": ["id", "name", "ram", "disk"]}}} - def index(self, req, **kw): + def index(self, req): """Return all flavors in brief.""" return dict(flavors=[dict(id=flavor['id'], name=flavor['name']) for flavor in self.detail(req)['flavors']]) - def detail(self, req, **kw): + def detail(self, req): """Return all flavors in detail.""" items = [self.show(req, id)['flavor'] for id in self._all_ids(req)] return dict(flavors=items) - def show(self, req, id, **kw): + def show(self, req, id): """Return data about the given flavor id.""" ctxt = req.environ['nova.context'] values = db.instance_type_get_by_flavor_id(ctxt, id) diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index 5bc5b9978..cf85a496f 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -115,14 +115,14 @@ class Controller(wsgi.Controller): def __init__(self): self._service = utils.import_object(FLAGS.image_service) - def index(self, req, **kw): + def index(self, req): """Return all public images in brief""" items = self._service.index(req.environ['nova.context']) items = common.limited(items, req) items = [_filter_keys(item, ('id', 'name')) for item in items] return dict(images=items) - def detail(self, req, **kw): + def detail(self, req): """Return all public images in detail""" try: items = self._service.detail(req.environ['nova.context']) @@ -136,7 +136,7 @@ class Controller(wsgi.Controller): items = [_translate_status(item) for item in items] return dict(images=items) - def show(self, req, id, **kw): + def show(self, req, id): """Return data about the given image id""" image_id = common.get_image_id_from_image_hash(self._service, req.environ['nova.context'], id) @@ -145,11 +145,11 @@ class Controller(wsgi.Controller): _convert_image_id_to_hash(image) return dict(image=image) - def delete(self, req, id, **kw): + def delete(self, req, id): # Only public images are supported for now. raise faults.Fault(exc.HTTPNotFound()) - def create(self, req, **kw): + def create(self, req): context = req.environ['nova.context'] env = self._deserialize(req.body, req) instance_id = env["image"]["serverId"] @@ -160,7 +160,7 @@ class Controller(wsgi.Controller): return dict(image=image_meta) - def update(self, req, id, **kw): + def update(self, req, id): # Users may not modify public images, and that's all that # we support for now. raise faults.Fault(exc.HTTPNotFound()) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 54060c2bb..c2bf42b72 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -105,11 +105,11 @@ class Controller(wsgi.Controller): self._image_service = utils.import_object(FLAGS.image_service) super(Controller, self).__init__() - def index(self, req, **kw): + def index(self, req): """ Returns a list of server names and ids for a given user """ return self._items(req, entity_maker=_translate_keys) - def detail(self, req, **kw): + def detail(self, req): """ Returns a list of server details for a given user """ return self._items(req, entity_maker=_translate_detail_keys) @@ -123,7 +123,7 @@ class Controller(wsgi.Controller): res = [entity_maker(inst)['server'] for inst in limited_list] return dict(servers=res) - def show(self, req, id, **kw): + def show(self, req, id): """ Returns server details by server id """ try: instance = self.compute_api.get(req.environ['nova.context'], id) @@ -131,7 +131,7 @@ class Controller(wsgi.Controller): except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) - def delete(self, req, id, **kw): + def delete(self, req, id): """ Destroys a server """ try: self.compute_api.delete(req.environ['nova.context'], id) @@ -139,7 +139,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def create(self, req, **kw): + def create(self, req): """ Creates a new server for a given user """ env = self._deserialize(req.body, req) if not env: @@ -180,7 +180,7 @@ class Controller(wsgi.Controller): onset_files=env.get('onset_files', [])) return _translate_keys(instances[0]) - def update(self, req, id, **kw): + def update(self, req, id): """ Updates the server name or password """ inst_dict = self._deserialize(req.body, req) if not inst_dict: @@ -202,7 +202,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPNoContent() - def action(self, req, id, **kw): + def action(self, req, id): """Multi-purpose method used to reboot, rebuild, or resize a server""" @@ -267,7 +267,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def lock(self, req, id, **kw): + def lock(self, req, id): """ lock the instance with id admin only operation @@ -282,7 +282,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def unlock(self, req, id, **kw): + def unlock(self, req, id): """ unlock the instance with id admin only operation @@ -297,7 +297,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def get_lock(self, req, id, **kw): + def get_lock(self, req, id): """ return the boolean state of (instance with id)'s lock @@ -311,7 +311,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def reset_network(self, req, id, **kw): + def reset_network(self, req, id): """ Reset networking on an instance (admin only). @@ -325,7 +325,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def inject_network_info(self, req, id, **kw): + def inject_network_info(self, req, id): """ Inject network info for an instance (admin only). @@ -339,7 +339,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def pause(self, req, id, **kw): + def pause(self, req, id): """ Permit Admins to Pause the server. """ ctxt = req.environ['nova.context'] try: @@ -350,7 +350,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def unpause(self, req, id, **kw): + def unpause(self, req, id): """ Permit Admins to Unpause the server. """ ctxt = req.environ['nova.context'] try: @@ -361,7 +361,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def suspend(self, req, id, **kw): + def suspend(self, req, id): """permit admins to suspend the server""" context = req.environ['nova.context'] try: @@ -372,7 +372,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def resume(self, req, id, **kw): + def resume(self, req, id): """permit admins to resume the server from suspend""" context = req.environ['nova.context'] try: @@ -383,7 +383,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def rescue(self, req, id, **kw): + def rescue(self, req, id): """Permit users to rescue the server.""" context = req.environ["nova.context"] try: @@ -394,7 +394,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def unrescue(self, req, id, **kw): + def unrescue(self, req, id): """Permit users to unrescue the server.""" context = req.environ["nova.context"] try: @@ -405,7 +405,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) return exc.HTTPAccepted() - def get_ajax_console(self, req, id, **kw): + def get_ajax_console(self, req, id): """ Returns a url to an instance's ajaxterm console. """ try: self.compute_api.get_ajax_console(req.environ['nova.context'], @@ -414,12 +414,12 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def diagnostics(self, req, id, **kw): + def diagnostics(self, req, id): """Permit Admins to retrieve server diagnostics.""" ctxt = req.environ["nova.context"] return self.compute_api.get_diagnostics(ctxt, id) - def actions(self, req, id, **kw): + def actions(self, req, id): """Permit Admins to retrieve server actions.""" ctxt = req.environ["nova.context"] items = self.compute_api.get_actions(ctxt, id) diff --git a/nova/api/openstack/shared_ip_groups.py b/nova/api/openstack/shared_ip_groups.py index e3c917749..5d78f9377 100644 --- a/nova/api/openstack/shared_ip_groups.py +++ b/nova/api/openstack/shared_ip_groups.py @@ -40,26 +40,26 @@ class Controller(wsgi.Controller): 'attributes': { 'sharedIpGroup': []}}} - def index(self, req, **kw): + def index(self, req): """ Returns a list of Shared IP Groups for the user """ return dict(sharedIpGroups=[]) - def show(self, req, id, **kw): + def show(self, req, id): """ Shows in-depth information on a specific Shared IP Group """ return _translate_keys({}) - def update(self, req, id, **kw): + def update(self, req, id): """ You can't update a Shared IP Group """ raise faults.Fault(exc.HTTPNotImplemented()) - def delete(self, req, id, **kw): + def delete(self, req, id): """ Deletes a Shared IP Group """ raise faults.Fault(exc.HTTPNotImplemented()) - def detail(self, req, **kw): + def detail(self, req): """ Returns a complete list of Shared IP Groups """ return _translate_detail_keys({}) - def create(self, req, **kw): + def create(self, req): """ Creates a new Shared IP group """ raise faults.Fault(exc.HTTPNotImplemented()) diff --git a/nova/api/openstack/users.py b/nova/api/openstack/users.py index ae3bf7791..83ebec964 100644 --- a/nova/api/openstack/users.py +++ b/nova/api/openstack/users.py @@ -50,28 +50,28 @@ class Controller(wsgi.Controller): if not context.is_admin: raise exception.NotAuthorized(_("Not admin user")) - def index(self, req, **kw): + def index(self, req): """Return all users in brief""" users = self.manager.get_users() users = common.limited(users, req) users = [_translate_keys(user) for user in users] return dict(users=users) - def detail(self, req, **kw): + def detail(self, req): """Return all users in detail""" return self.index(req) - def show(self, req, id, **kw): + def show(self, req, id): """Return data about the given user id""" user = self.manager.get_user(id) return dict(user=_translate_keys(user)) - def delete(self, req, id, **kw): + def delete(self, req, id): self._check_admin(req.environ['nova.context']) self.manager.delete_user(id) return {} - def create(self, req, **kw): + def create(self, req): self._check_admin(req.environ['nova.context']) env = self._deserialize(req.body, req) is_admin = env['user'].get('admin') in ('T', 'True', True) @@ -81,7 +81,7 @@ class Controller(wsgi.Controller): user = self.manager.create_user(name, access, secret, is_admin) return dict(user=_translate_keys(user)) - def update(self, req, id, **kw): + def update(self, req, id): self._check_admin(req.environ['nova.context']) env = self._deserialize(req.body, req) is_admin = env['user'].get('admin') diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index 30bf2b67b..d5206da20 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -43,35 +43,35 @@ class Controller(wsgi.Controller): "attributes": { "zone": ["id", "api_url"]}}} - def index(self, req, **kw): + def index(self, req): """Return all zones in brief""" items = db.zone_get_all(req.environ['nova.context']) items = common.limited(items, req) items = [_scrub_zone(item) for item in items] return dict(zones=items) - def detail(self, req, **kw): + def detail(self, req): """Return all zones in detail""" return self.index(req) - def show(self, req, id, **kw): + def show(self, req, id): """Return data about the given zone id""" zone_id = int(id) zone = db.zone_get(req.environ['nova.context'], zone_id) return dict(zone=_scrub_zone(zone)) - def delete(self, req, id, **kw): + def delete(self, req, id): zone_id = int(id) db.zone_delete(req.environ['nova.context'], zone_id) return {} - def create(self, req, **kw): + def create(self, req): context = req.environ['nova.context'] env = self._deserialize(req.body, req) zone = db.zone_create(context, env["zone"]) return dict(zone=_scrub_zone(zone)) - def update(self, req, id, **kw): + def update(self, req, id): context = req.environ['nova.context'] env = self._deserialize(req.body, req) zone_id = int(id) -- cgit From e44f085ed464a3397e3bf89a3e5355e538c71a65 Mon Sep 17 00:00:00 2001 From: Ricardo Carrillo Cruz Date: Wed, 9 Mar 2011 19:16:26 +0100 Subject: Fixed pep8 issues --- nova/db/api.py | 7 +++++-- nova/db/sqlalchemy/api.py | 7 +++++-- nova/network/manager.py | 4 ++-- 3 files changed, 12 insertions(+), 6 deletions(-) (limited to 'nova') diff --git a/nova/db/api.py b/nova/db/api.py index 7ad99c1f4..5c34a02e4 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -459,6 +459,7 @@ def network_associate(context, project_id): """Associate a free network to a project.""" return IMPL.network_associate(context, project_id) + def network_count(context): """Return the number of networks.""" return IMPL.network_count(context) @@ -488,9 +489,9 @@ def network_create_safe(context, values): """ return IMPL.network_create_safe(context, values) + def network_delete_safe(context, network_id): - """Delete network with key network_id - + """Delete network with key network_id. This method assumes that the network is not associated with any project """ return IMPL.network_delete_safe(context, network_id) @@ -531,10 +532,12 @@ def network_get_by_bridge(context, bridge): """Get a network by bridge or raise if it does not exist.""" return IMPL.network_get_by_bridge(context, bridge) + def network_get_by_cidr(context, cidr): """Get a network by cidr or raise if it does not exist""" return IMPL.network_get_by_cidr(context, cidr) + def network_get_by_instance(context, instance_id): """Get a network by instance id or raise if it does not exist.""" return IMPL.network_get_by_instance(context, instance_id) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 90730d325..3a1162a17 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1042,12 +1042,14 @@ def network_create_safe(context, values): except IntegrityError: return None + @require_admin_context def network_delete_safe(context, network_id): session = get_session() with session.begin(): - network_ref = network_get(context, network_id=network_id, session=session) - session.delete(network_ref) + network_ref = network_get(context, network_id=network_id, \ + session=session) + session.delete(network_ref) @require_admin_context @@ -1134,6 +1136,7 @@ def network_get_by_cidr(context, cidr): cidr) return result + @require_admin_context def network_get_by_instance(_context, instance_id): session = get_session() diff --git a/nova/network/manager.py b/nova/network/manager.py index 39da031eb..3dfc48934 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -568,11 +568,11 @@ class VlanManager(NetworkManager): network_ref = db.network_get_by_cidr(context, cidr) except exception.NotFound: pass - + if network_ref is not None: raise ValueError(_('Network with cidr %s already exists' % cidr)) - + network_ref = self.db.network_create_safe(context, net) if network_ref: self._create_fixed_ips(context, network_ref['id']) -- cgit From 1166e16d08769222189e31e6de1c6019495fc743 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 9 Mar 2011 13:54:30 -0500 Subject: move my tests into their own testcase --- nova/tests/api/openstack/test_servers.py | 447 +++++++++++++++++-------------- 1 file changed, 243 insertions(+), 204 deletions(-) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index f2fbb04af..b0f888766 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -238,210 +238,6 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) - def _setup_mock_compute_api_for_personality(self): - - class MockComputeAPI(object): - - def __init__(self): - self.personality_files = None - - def create(self, *args, **kwargs): - if 'personality_files' in kwargs: - self.personality_files = kwargs['personality_files'] - else: - self.personality_files = None - return [{'id': '1234', 'display_name': 'fakeinstance'}] - - def make_stub_method(canned_return): - def stub_method(*args, **kwargs): - return canned_return - return stub_method - - compute_api = MockComputeAPI() - self.stubs.Set(nova.compute, 'API', make_stub_method(compute_api)) - self.stubs.Set(nova.api.openstack.servers.Controller, - '_get_kernel_ramdisk_from_image', make_stub_method((1, 1))) - self.stubs.Set(nova.api.openstack.common, - 'get_image_id_from_image_hash', make_stub_method(2)) - return compute_api - - def _create_personality_request_dict(self, personality_files): - server = {} - server['name'] = 'new-server-test' - server['imageId'] = 1 - server['flavorId'] = 1 - if personality_files is not None: - personalities = [] - for path, contents in personality_files: - personalities.append({'path': path, 'contents': contents}) - server['personality'] = personalities - return {'server': server} - - def _get_create_request_json(self, body_dict): - req = webob.Request.blank('/v1.0/servers') - req.content_type = 'application/json' - req.method = 'POST' - req.body = json.dumps(body_dict) - return req - - def _run_create_instance_with_mock_compute_api(self, request): - compute_api = self._setup_mock_compute_api_for_personality() - response = request.get_response(fakes.wsgi_app()) - return compute_api, response - - def _format_xml_request_body(self, body_dict): - server = body_dict['server'] - body_parts = [] - body_parts.extend([ - '', - '' % ( - server['name'], server['imageId'], server['flavorId'])]) - if 'metadata' in server: - metadata = server['metadata'] - body_parts.append('') - for item in metadata.iteritems(): - body_parts.append('%s' % item) - body_parts.append('') - if 'personality' in server: - personalities = server['personality'] - body_parts.append('') - for file in personalities: - item = (file['path'], file['contents']) - body_parts.append('%s' % item) - body_parts.append('') - body_parts.append('') - return ''.join(body_parts) - - def _get_create_request_xml(self, body_dict): - req = webob.Request.blank('/v1.0/servers') - req.content_type = 'application/xml' - req.method = 'POST' - req.body = self._format_xml_request_body(body_dict) - return req - - def _create_instance_with_personality_json(self, personality): - body_dict = self._create_personality_request_dict(personality) - request = self._get_create_request_json(body_dict) - compute_api, response = \ - self._run_create_instance_with_mock_compute_api(request) - return request, response, compute_api.personality_files - - def _create_instance_with_personality_xml(self, personality): - body_dict = self._create_personality_request_dict(personality) - request = self._get_create_request_xml(body_dict) - compute_api, response = \ - self._run_create_instance_with_mock_compute_api(request) - return request, response, compute_api.personality_files - - def test_create_instance_with_no_personality(self): - request, response, personality_files = \ - self._create_instance_with_personality_json(personality=None) - self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, []) - - def test_create_instance_with_no_personality_xml(self): - request, response, personality_files = \ - self._create_instance_with_personality_xml(personality=None) - self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, []) - - def test_create_instance_with_personality(self): - path = '/my/file/path' - contents = '#!/bin/bash\necho "Hello, World!"\n' - b64contents = base64.b64encode(contents) - personality = [(path, b64contents)] - request, response, personality_files = \ - self._create_instance_with_personality_json(personality) - self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, [(path, contents)]) - - def test_create_instance_with_personality_xml(self): - path = '/my/file/path' - contents = '#!/bin/bash\necho "Hello, World!"\n' - b64contents = base64.b64encode(contents) - personality = [(path, b64contents)] - request, response, personality_files = \ - self._create_instance_with_personality_xml(personality) - self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, [(path, contents)]) - - def test_create_instance_with_personality_no_path(self): - personality = [('/remove/this/path', - base64.b64encode('my\n\file\ncontents'))] - body_dict = self._create_personality_request_dict(personality) - del body_dict['server']['personality'][0]['path'] - request = self._get_create_request_json(body_dict) - compute_api, response = \ - self._run_create_instance_with_mock_compute_api(request) - self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) - - def _test_create_instance_with_personality_no_path_xml(self): - personality = [('/remove/this/path', - base64.b64encode('my\n\file\ncontents'))] - body_dict = self._create_personality_request_dict(personality) - request = self._get_create_request_xml(body_dict) - request.body = request.body.replace(' path="/remove/this/path"', '') - compute_api, response = \ - self._run_create_instance_with_mock_compute_api(request) - self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) - - def test_create_instance_with_personality_no_contents(self): - personality = [('/test/path', - base64.b64encode('remove\nthese\ncontents'))] - body_dict = self._create_personality_request_dict(personality) - del body_dict['server']['personality'][0]['contents'] - request = self._get_create_request_json(body_dict) - compute_api, response = \ - self._run_create_instance_with_mock_compute_api(request) - self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) - - def test_create_instance_with_personality_not_a_list(self): - personality = [('/test/path', base64.b64encode('test\ncontents\n'))] - body_dict = self._create_personality_request_dict(personality) - body_dict['server']['personality'] = \ - body_dict['server']['personality'][0] - request = self._get_create_request_json(body_dict) - compute_api, response = \ - self._run_create_instance_with_mock_compute_api(request) - self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) - - def test_create_instance_with_personality_with_non_b64_content(self): - path = '/my/file/path' - contents = '#!/bin/bash\necho "Oh no!"\n' - personality = [(path, contents)] - request, response, personality_files = \ - self._create_instance_with_personality_json(personality) - self.assertEquals(response.status_int, 400) - self.assertEquals(personality_files, None) - - def test_create_instance_with_three_personalities(self): - files = [ - ('/etc/sudoers', 'ALL ALL=NOPASSWD: ALL\n'), - ('/etc/motd', 'Enjoy your root access!\n'), - ('/etc/dovecot.conf', 'dovecot\nconfig\nstuff\n'), - ] - personality = [] - for path, content in files: - personality.append((path, base64.b64encode(content))) - request, response, personality_files = \ - self._create_instance_with_personality_json(personality) - self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, files) - - def test_create_instance_personality_empty_content(self): - path = '/my/file/path' - contents = '' - personality = [(path, contents)] - request, response, personality_files = \ - self._create_instance_with_personality_json(personality) - self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, [(path, contents)]) - def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' @@ -980,5 +776,248 @@ b25zLiINCg0KLVJpY2hhcmQgQmFjaA==""", self.assertEqual(request, expected) +class TestServerInstanceCreation(test.TestCase): + + def setUp(self): + super(TestServerInstanceCreation, self).setUp() + self.stubs = stubout.StubOutForTesting() + fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthDatabase.data = {} + fakes.stub_out_networking(self.stubs) + fakes.stub_out_rate_limiting(self.stubs) + fakes.stub_out_auth(self.stubs) + fakes.stub_out_key_pair_funcs(self.stubs) + fakes.stub_out_image_service(self.stubs) + self.stubs.Set(nova.db.api, 'instance_get_all', return_servers) + self.stubs.Set(nova.db.api, 'instance_get', return_server) + self.stubs.Set(nova.db.api, 'instance_get_all_by_user', + return_servers) + self.stubs.Set(nova.db.api, 'instance_add_security_group', + return_security_group) + self.stubs.Set(nova.db.api, 'instance_update', instance_update) + self.stubs.Set(nova.db.api, 'instance_get_fixed_address', + instance_address) + self.stubs.Set(nova.db.api, 'instance_get_floating_address', + instance_address) + self.stubs.Set(nova.compute.API, 'pause', fake_compute_api) + self.stubs.Set(nova.compute.API, 'unpause', fake_compute_api) + self.stubs.Set(nova.compute.API, 'suspend', fake_compute_api) + self.stubs.Set(nova.compute.API, 'resume', fake_compute_api) + self.stubs.Set(nova.compute.API, "get_diagnostics", fake_compute_api) + self.stubs.Set(nova.compute.API, "get_actions", fake_compute_api) + self.allow_admin = FLAGS.allow_admin_api + + self.webreq = common.webob_factory('/v1.0/servers') + + def tearDown(self): + self.stubs.UnsetAll() + FLAGS.allow_admin_api = self.allow_admin + super(TestServerInstanceCreation, self).tearDown() + + def _setup_mock_compute_api_for_personality(self): + + class MockComputeAPI(object): + + def __init__(self): + self.personality_files = None + + def create(self, *args, **kwargs): + if 'personality_files' in kwargs: + self.personality_files = kwargs['personality_files'] + else: + self.personality_files = None + return [{'id': '1234', 'display_name': 'fakeinstance'}] + + def make_stub_method(canned_return): + def stub_method(*args, **kwargs): + return canned_return + return stub_method + + compute_api = MockComputeAPI() + self.stubs.Set(nova.compute, 'API', make_stub_method(compute_api)) + self.stubs.Set(nova.api.openstack.servers.Controller, + '_get_kernel_ramdisk_from_image', make_stub_method((1, 1))) + self.stubs.Set(nova.api.openstack.common, + 'get_image_id_from_image_hash', make_stub_method(2)) + return compute_api + + def _create_personality_request_dict(self, personality_files): + server = {} + server['name'] = 'new-server-test' + server['imageId'] = 1 + server['flavorId'] = 1 + if personality_files is not None: + personalities = [] + for path, contents in personality_files: + personalities.append({'path': path, 'contents': contents}) + server['personality'] = personalities + return {'server': server} + + def _get_create_request_json(self, body_dict): + req = webob.Request.blank('/v1.0/servers') + req.content_type = 'application/json' + req.method = 'POST' + req.body = json.dumps(body_dict) + return req + + def _run_create_instance_with_mock_compute_api(self, request): + compute_api = self._setup_mock_compute_api_for_personality() + response = request.get_response(fakes.wsgi_app()) + return compute_api, response + + def _format_xml_request_body(self, body_dict): + server = body_dict['server'] + body_parts = [] + body_parts.extend([ + '', + '' % ( + server['name'], server['imageId'], server['flavorId'])]) + if 'metadata' in server: + metadata = server['metadata'] + body_parts.append('') + for item in metadata.iteritems(): + body_parts.append('%s' % item) + body_parts.append('') + if 'personality' in server: + personalities = server['personality'] + body_parts.append('') + for file in personalities: + item = (file['path'], file['contents']) + body_parts.append('%s' % item) + body_parts.append('') + body_parts.append('') + return ''.join(body_parts) + + def _get_create_request_xml(self, body_dict): + req = webob.Request.blank('/v1.0/servers') + req.content_type = 'application/xml' + req.method = 'POST' + req.body = self._format_xml_request_body(body_dict) + return req + + def _create_instance_with_personality_json(self, personality): + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + return request, response, compute_api.personality_files + + def _create_instance_with_personality_xml(self, personality): + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_xml(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + return request, response, compute_api.personality_files + + def test_create_instance_with_no_personality(self): + request, response, personality_files = \ + self._create_instance_with_personality_json(personality=None) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, []) + + def test_create_instance_with_no_personality_xml(self): + request, response, personality_files = \ + self._create_instance_with_personality_xml(personality=None) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, []) + + def test_create_instance_with_personality(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Hello, World!"\n' + b64contents = base64.b64encode(contents) + personality = [(path, b64contents)] + request, response, personality_files = \ + self._create_instance_with_personality_json(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, [(path, contents)]) + + def test_create_instance_with_personality_xml(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Hello, World!"\n' + b64contents = base64.b64encode(contents) + personality = [(path, b64contents)] + request, response, personality_files = \ + self._create_instance_with_personality_xml(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, [(path, contents)]) + + def test_create_instance_with_personality_no_path(self): + personality = [('/remove/this/path', + base64.b64encode('my\n\file\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + del body_dict['server']['personality'][0]['path'] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + + def _test_create_instance_with_personality_no_path_xml(self): + personality = [('/remove/this/path', + base64.b64encode('my\n\file\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_xml(body_dict) + request.body = request.body.replace(' path="/remove/this/path"', '') + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + + def test_create_instance_with_personality_no_contents(self): + personality = [('/test/path', + base64.b64encode('remove\nthese\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + del body_dict['server']['personality'][0]['contents'] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + + def test_create_instance_with_personality_not_a_list(self): + personality = [('/test/path', base64.b64encode('test\ncontents\n'))] + body_dict = self._create_personality_request_dict(personality) + body_dict['server']['personality'] = \ + body_dict['server']['personality'][0] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + + def test_create_instance_with_personality_with_non_b64_content(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Oh no!"\n' + personality = [(path, contents)] + request, response, personality_files = \ + self._create_instance_with_personality_json(personality) + self.assertEquals(response.status_int, 400) + self.assertEquals(personality_files, None) + + def test_create_instance_with_three_personalities(self): + files = [ + ('/etc/sudoers', 'ALL ALL=NOPASSWD: ALL\n'), + ('/etc/motd', 'Enjoy your root access!\n'), + ('/etc/dovecot.conf', 'dovecot\nconfig\nstuff\n'), + ] + personality = [] + for path, content in files: + personality.append((path, base64.b64encode(content))) + request, response, personality_files = \ + self._create_instance_with_personality_json(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, files) + + def test_create_instance_personality_empty_content(self): + path = '/my/file/path' + contents = '' + personality = [(path, contents)] + request, response, personality_files = \ + self._create_instance_with_personality_json(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, [(path, contents)]) + + if __name__ == "__main__": unittest.main() -- cgit From 3999bb363501c6587f75255333094c9e61bf1828 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 9 Mar 2011 14:07:33 -0500 Subject: remove unneeded stubs --- nova/tests/api/openstack/test_servers.py | 22 ---------------------- 1 file changed, 22 deletions(-) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index b0f888766..c67ecdaae 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -783,32 +783,10 @@ class TestServerInstanceCreation(test.TestCase): self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} - fakes.stub_out_networking(self.stubs) - fakes.stub_out_rate_limiting(self.stubs) fakes.stub_out_auth(self.stubs) fakes.stub_out_key_pair_funcs(self.stubs) - fakes.stub_out_image_service(self.stubs) - self.stubs.Set(nova.db.api, 'instance_get_all', return_servers) - self.stubs.Set(nova.db.api, 'instance_get', return_server) - self.stubs.Set(nova.db.api, 'instance_get_all_by_user', - return_servers) - self.stubs.Set(nova.db.api, 'instance_add_security_group', - return_security_group) - self.stubs.Set(nova.db.api, 'instance_update', instance_update) - self.stubs.Set(nova.db.api, 'instance_get_fixed_address', - instance_address) - self.stubs.Set(nova.db.api, 'instance_get_floating_address', - instance_address) - self.stubs.Set(nova.compute.API, 'pause', fake_compute_api) - self.stubs.Set(nova.compute.API, 'unpause', fake_compute_api) - self.stubs.Set(nova.compute.API, 'suspend', fake_compute_api) - self.stubs.Set(nova.compute.API, 'resume', fake_compute_api) - self.stubs.Set(nova.compute.API, "get_diagnostics", fake_compute_api) - self.stubs.Set(nova.compute.API, "get_actions", fake_compute_api) self.allow_admin = FLAGS.allow_admin_api - self.webreq = common.webob_factory('/v1.0/servers') - def tearDown(self): self.stubs.UnsetAll() FLAGS.allow_admin_api = self.allow_admin -- cgit From 23369a63f4b74fb64bf57554a3fd8b15e3e2b49c Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 14:31:23 -0500 Subject: Fixes uses of process_input --- nova/utils.py | 4 ++-- nova/virt/disk.py | 4 ++-- nova/virt/libvirt_conn.py | 11 ++++------- 3 files changed, 8 insertions(+), 11 deletions(-) (limited to 'nova') diff --git a/nova/utils.py b/nova/utils.py index 7ddf056ea..0937522ec 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -131,7 +131,7 @@ def fetchfile(url, target): def execute(*cmd, **kwargs): process_input=kwargs.get('process_input', None) addl_env=kwargs.get('addl_env', None) - check_exit_code=kwargs.get('check_exit_code', True) + check_exit_code=kwargs.get('check_exit_code', 0) stdin=kwargs.get('stdin', subprocess.PIPE) stdout=kwargs.get('stdout', subprocess.PIPE) stderr=kwargs.get('stderr', subprocess.PIPE) @@ -151,7 +151,7 @@ def execute(*cmd, **kwargs): obj.stdin.close() if obj.returncode: LOG.debug(_("Result was %s") % obj.returncode) - if check_exit_code and obj.returncode != 0: + if check_exit_code is not None and obj.returncode != check_exit_code: (stdout, stderr) = result raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, diff --git a/nova/virt/disk.py b/nova/virt/disk.py index 203517275..a54cda003 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -175,8 +175,8 @@ def _inject_key_into_fs(key, fs): utils.execute('sudo', 'chown', 'root', sshdir) utils.execute('sudo', 'chmod', '700', sshdir) keyfile = os.path.join(sshdir, 'authorized_keys') - # TODO:EWINDISCH: not sure about the following /w execv patch - utils.execute('sudo', 'tee', '-a', keyfile, '\n' + key.strip() + '\n') + utils.execute('sudo', 'tee', '-a', keyfile, + process_input='\n' + key.strip() + '\n') def _inject_net_into_fs(net, fs): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 76f31f91a..6b555ecbb 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -485,13 +485,10 @@ class LibvirtConnection(object): port = random.randint(int(start_port), int(end_port)) # netcat will exit with 0 only if the port is in use, # so a nonzero return value implies it is unused - - # TODO(ewindisch): broken /w execvp patch. - # subprocess lets us do this, but utils.execute - # abstracts it away from us - cmd = 'netcat', '0.0.0.0', port, '-w', '1', ' Date: Wed, 9 Mar 2011 14:38:34 -0500 Subject: rearrange functions and add docstrings --- nova/api/openstack/servers.py | 83 ++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 33 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 4a18d870c..419a001fb 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -141,39 +141,6 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def _get_personality_files(self, personality): - """ - Create a list of personality files from the personality attribute - - At this time, personality_files must be formatted as a list of - (file_path, file_content) pairs for compatibility with the - underlying compute service. - """ - personality_files = [] - for item in personality: - try: - path = item['path'] - contents = item['contents'] - except KeyError as key: - expl = 'Bad personality format: missing %s' % key - raise exc.HTTPBadRequest(explanation=expl) - except TypeError: - raise exc.HTTPBadRequest(explanation='Bad personality format') - try: - contents = base64.b64decode(contents) - except TypeError: - msg = 'Personality content for %s cannot be decoded' % path - raise exc.HTTPBadRequest(explanation=msg) - personality_files.append((path, contents)) - return personality_files - - def _deserialize_create(self, request): - if request.content_type == "application/xml": - deserializer = ServerCreateRequestXMLDeserializer() - return deserializer.deserialize(request.body) - else: - return self._deserialize(request.body, request) - def create(self, req): """ Creates a new server for a given user """ env = self._deserialize_create(req) @@ -218,6 +185,44 @@ class Controller(wsgi.Controller): personality_files=personality_files) return _translate_keys(instances[0]) + def _deserialize_create(self, request): + """ + Deserialize a create request + + Overrides normal behavior in the case of xml content + """ + if request.content_type == "application/xml": + deserializer = ServerCreateRequestXMLDeserializer() + return deserializer.deserialize(request.body) + else: + return self._deserialize(request.body, request) + + def _get_personality_files(self, personality): + """ + Create a list of personality files from the personality attribute + + At this time, personality_files must be formatted as a list of + (file_path, file_content) pairs for compatibility with the + underlying compute service. + """ + personality_files = [] + for item in personality: + try: + path = item['path'] + contents = item['contents'] + except KeyError as key: + expl = 'Bad personality format: missing %s' % key + raise exc.HTTPBadRequest(explanation=expl) + except TypeError: + raise exc.HTTPBadRequest(explanation='Bad personality format') + try: + contents = base64.b64decode(contents) + except TypeError: + msg = 'Personality content for %s cannot be decoded' % path + raise exc.HTTPBadRequest(explanation=msg) + personality_files.append((path, contents)) + return personality_files + def update(self, req, id): """ Updates the server name or password """ inst_dict = self._deserialize(req.body, req) @@ -507,13 +512,21 @@ class Controller(wsgi.Controller): class ServerCreateRequestXMLDeserializer(object): + """ + Deserializer to handle xml-formatted server create requests. + + Handles standard server attributes as well as optional metadata + and personality attributes + """ def deserialize(self, string): + """Deserialize an xml-formatted server create request""" dom = minidom.parseString(string) server = self._extract_server(dom) return {'server': server} def _extract_server(self, node): + """Marshal the server attribute of a parsed request""" server = {} server_node = self._find_first_child_named(node, 'server') for attr in ["name", "imageId", "flavorId"]: @@ -527,6 +540,7 @@ class ServerCreateRequestXMLDeserializer(object): return server def _extract_metadata(self, server_node): + """Marshal the metadata attribute of a parsed request""" metadata_node = self._find_first_child_named(server_node, "metadata") if metadata_node is None: return None @@ -537,6 +551,7 @@ class ServerCreateRequestXMLDeserializer(object): return metadata def _extract_personality(self, server_node): + """Marshal the personality attribute of a parsed request""" personality_node = \ self._find_first_child_named(server_node, "personality") if personality_node is None: @@ -551,12 +566,14 @@ class ServerCreateRequestXMLDeserializer(object): return personality def _find_first_child_named(self, parent, name): + """Search a nodes children for the first child with a given name""" for node in parent.childNodes: if node.nodeName == name: return node return None def _extract_text(self, node): + """Get the text field contained by the given node""" if len(node.childNodes) == 1: child = node.childNodes[0] if child.nodeType == child.TEXT_NODE: -- cgit From a9bd1b456332aaf5f6ab9942979485f2192b6f3e Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Wed, 9 Mar 2011 14:53:44 -0500 Subject: Add password parameter to the set_admin_password call in the compute api. Updated servers password to use this parameter. --- nova/api/openstack/servers.py | 3 ++- nova/compute/api.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 7222285e0..41166f810 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -183,7 +183,8 @@ class Controller(wsgi.Controller): password = "%s%s" % (server['server']['name'][:4], utils.generate_password(12)) server['server']['adminPass'] = password - self.compute_api.set_admin_password(context, server['server']['id']) + self.compute_api.set_admin_password(context, server['server']['id'], + password) return server def update(self, req, id): diff --git a/nova/compute/api.py b/nova/compute/api.py index 33d25fc4b..a0bb2cf04 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -498,9 +498,10 @@ class API(base.Base): """Unrescue the given instance.""" self._cast_compute_message('unrescue_instance', context, instance_id) - def set_admin_password(self, context, instance_id): + def set_admin_password(self, context, instance_id, password=None): """Set the root/admin password for the given instance.""" - self._cast_compute_message('set_admin_password', context, instance_id) + self._cast_compute_message('set_admin_password', context, instance_id, + password) def inject_file(self, context, instance_id): """Write a file to the given instance.""" -- cgit From 3f723bcf54b4d779c66373dc8f69f43923dd586a Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Wed, 9 Mar 2011 15:08:11 -0500 Subject: renaming wsgi.Request.best_match to best_match_content_type; correcting calls to that function in code from trunk --- nova/api/direct.py | 2 +- nova/api/openstack/__init__.py | 2 +- nova/api/openstack/faults.py | 2 +- nova/api/openstack/servers.py | 2 +- nova/tests/api/openstack/common.py | 1 + nova/tests/api/test_wsgi.py | 18 +++++++++--------- nova/wsgi.py | 4 ++-- 7 files changed, 16 insertions(+), 15 deletions(-) (limited to 'nova') diff --git a/nova/api/direct.py b/nova/api/direct.py index 1d699f947..dfca250e0 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -206,7 +206,7 @@ class ServiceWrapper(wsgi.Controller): params = dict([(str(k), v) for (k, v) in params.iteritems()]) result = method(context, **params) if type(result) is dict or type(result) is list: - return self._serialize(result, req.best_match()) + return self._serialize(result, req.best_match_content_type()) else: return result diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 6e1a2a06c..197fcc619 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -125,5 +125,5 @@ class Versions(wsgi.Application): "application/xml": { "attributes": dict(version=["status", "id"])}} - content_type = req.best_match() + content_type = req.best_match_content_type() return wsgi.Serializer(metadata).serialize(response, content_type) diff --git a/nova/api/openstack/faults.py b/nova/api/openstack/faults.py index 075fdb997..2fd733299 100644 --- a/nova/api/openstack/faults.py +++ b/nova/api/openstack/faults.py @@ -58,6 +58,6 @@ class Fault(webob.exc.HTTPException): # 'code' is an attribute on the fault tag itself metadata = {'application/xml': {'attributes': {fault_name: 'code'}}} serializer = wsgi.Serializer(metadata) - content_type = req.best_match() + content_type = req.best_match_content_type() self.wrapped_exc.body = serializer.serialize(fault_data, content_type) return self.wrapped_exc diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 8dd078a31..25c667532 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -217,7 +217,7 @@ class Controller(wsgi.Controller): 'rebuild': self._action_rebuild, } - input_dict = self._deserialize(req.body, req) + input_dict = self._deserialize(req.body, req.get_content_type()) for key in actions.keys(): if key in input_dict: return actions[key](input_dict, req, id) diff --git a/nova/tests/api/openstack/common.py b/nova/tests/api/openstack/common.py index 3f9c7d3cf..74bb8729a 100644 --- a/nova/tests/api/openstack/common.py +++ b/nova/tests/api/openstack/common.py @@ -28,6 +28,7 @@ def webob_factory(url): def web_request(url, method=None, body=None): req = webob.Request.blank("%s%s" % (base_url, url)) if method: + req.content_type = "application/json" req.method = method if body: req.body = json.dumps(body) diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index 7c0135656..b1a849cf9 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -139,48 +139,48 @@ class RequestTest(test.TestCase): def test_content_type_from_accept_xml(self): request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/xml" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/xml") request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/json" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/xml, application/json" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = \ "application/json; q=0.3, application/xml; q=0.9" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/xml") def test_content_type_from_query_extension(self): request = wsgi.Request.blank('/tests/123.xml') - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/xml") request = wsgi.Request.blank('/tests/123.json') - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") request = wsgi.Request.blank('/tests/123.invalid') - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") def test_content_type_accept_and_query_extension(self): request = wsgi.Request.blank('/tests/123.xml') request.headers["Accept"] = "application/json" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/xml") def test_content_type_accept_default(self): request = wsgi.Request.blank('/tests/123.unsupported') request.headers["Accept"] = "application/unsupported1" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") diff --git a/nova/wsgi.py b/nova/wsgi.py index c3e08522d..2d18da8fb 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -85,7 +85,7 @@ class Server(object): class Request(webob.Request): - def best_match(self): + def best_match_content_type(self): """ Determine the most acceptable content-type based on the query extension then the Accept header @@ -354,7 +354,7 @@ class Controller(object): result = method(**arg_dict) if type(result) is dict: - content_type = req.best_match() + content_type = req.best_match_content_type() body = self._serialize(result, content_type) response = webob.Response() -- cgit From fc9840bae6200c8f89fb8a3ba0ab45663c872b3c Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 15:33:20 -0500 Subject: execvp passes pep8 --- nova/console/xvp.py | 6 +++--- nova/crypto.py | 3 ++- nova/network/linux_net.py | 19 ++++++++++++------- nova/tests/test_virt.py | 2 +- nova/utils.py | 19 ++++++++++--------- nova/volume/driver.py | 4 ++-- 6 files changed, 30 insertions(+), 23 deletions(-) (limited to 'nova') diff --git a/nova/console/xvp.py b/nova/console/xvp.py index 271dffa54..68d8c8565 100644 --- a/nova/console/xvp.py +++ b/nova/console/xvp.py @@ -134,9 +134,9 @@ class XVPConsoleProxy(object): logging.debug(_("Starting xvp")) try: utils.execute('xvp', - '-p',FLAGS.console_xvp_pid, - '-c',FLAGS.console_xvp_conf, - '-l',FLAGS.console_xvp_log) + '-p', FLAGS.console_xvp_pid, + '-c', FLAGS.console_xvp_conf, + '-l', FLAGS.console_xvp_log) except exception.ProcessExecutionError, err: logging.error(_("Error starting xvp: %s") % err) diff --git a/nova/crypto.py b/nova/crypto.py index 20bb570a5..717ea0041 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -120,7 +120,8 @@ def generate_key_pair(bits=1024): # bio = M2Crypto.BIO.MemoryBuffer() # key.save_pub_key_bio(bio) # public_key = bio.read() - # public_key, err = execute('ssh-keygen', '-y', '-f', '/dev/stdin', private_key) + # public_key, err = execute('ssh-keygen', '-y', '-f', + # '/dev/stdin', private_key) return (private_key, public_key, fingerprint) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index b66a1adb7..228a4d9ea 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -68,7 +68,8 @@ def metadata_forward(): _confirm_rule("PREROUTING", '-t', 'nat', '-s', '0.0.0.0/0', '-d', '169.254.169.254/32', '-p', 'tcp', '-m', 'tcp', '--dport', '80', '-j', 'DNAT', - '--to-destination', '%s:%s' % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) + '--to-destination', + '%s:%s' % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) def init_host(): @@ -86,7 +87,8 @@ def init_host(): _execute('sudo', 'iptables', '-D', 'FORWARD', '-j', 'nova_forward', check_exit_code=False) _execute('sudo', 'iptables', '-A', 'FORWARD', '-j', 'nova_forward') - _execute('sudo', 'iptables', '-N', 'nova_output', check_exit_code=False) + _execute('sudo', 'iptables', '-N', 'nova_output', + check_exit_code=False) _execute('sudo', 'iptables', '-D', 'OUTPUT', '-j', 'nova_output', check_exit_code=False) _execute('sudo', 'iptables', '-A', 'OUTPUT', '-j', 'nova_output') @@ -220,7 +222,7 @@ def ensure_bridge(bridge, interface, net_attrs=None): # bridge for it to respond to reqests properly suffix = net_attrs['cidr'].rpartition('/')[2] out, err = _execute('sudo', 'ip', 'addr', 'add', - "%s/%s" % + "%s/%s" % (net_attrs['gateway'], suffix), 'brd', net_attrs['broadcast'], @@ -237,7 +239,8 @@ def ensure_bridge(bridge, interface, net_attrs=None): # bridge, then the bridge has to be in promiscuous # to forward packets properly. if(FLAGS.public_interface == bridge): - _execute('sudo', 'ip', 'link', 'set', 'dev', bridge, 'promisc', 'on') + _execute('sudo', 'ip', 'link', 'set', + 'dev', bridge, 'promisc', 'on') if interface: # NOTE(vish): This will break if there is already an ip on the # interface, so we move any ips to the bridge @@ -253,8 +256,10 @@ def ensure_bridge(bridge, interface, net_attrs=None): fields = line.split() if fields and fields[0] == "inet": params = ' '.join(fields[1:-1]) - _execute('sudo', 'ip', 'addr', 'del', params, 'dev', fields[-1]) - _execute('sudo', 'ip', 'addr', 'add', params, 'dev', bridge) + _execute('sudo', 'ip', 'addr', + 'del', params, 'dev', fields[-1]) + _execute('sudo', 'ip', 'addr', + 'add', params, 'dev', bridge) if gateway: _execute('sudo', 'route', 'add', '0.0.0.0', 'gw', gateway) out, err = _execute('sudo', 'brctl', 'addif', bridge, interface, @@ -397,7 +402,7 @@ def _device_exists(device): def _confirm_rule(chain, *cmd, **kwargs): - append=kwargs.get('append',False) + append = kwargs.get('append', False) """Delete and re-add iptables rule""" if FLAGS.use_nova_chains: chain = "nova_%s" % chain.lower() diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 7f1ad002e..dfa607f14 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -316,7 +316,7 @@ class IptablesFirewallTestCase(test.TestCase): # self.fw.add_instance(instance_ref) def fake_iptables_execute(*cmd, **kwargs): - process_input=kwargs.get('process_input', None) + process_input = kwargs.get('process_input', None) if cmd == ('sudo', 'ip6tables-save', '-t', 'filter'): return '\n'.join(self.in6_rules), None if cmd == ('sudo', 'iptables-save', '-t', 'filter'): diff --git a/nova/utils.py b/nova/utils.py index 0937522ec..3a4ec3c6a 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -40,7 +40,7 @@ import netaddr from eventlet import event from eventlet import greenthread from eventlet.green import subprocess - +None from nova import exception from nova.exception import ProcessExecutionError from nova import log as logging @@ -129,13 +129,13 @@ def fetchfile(url, target): def execute(*cmd, **kwargs): - process_input=kwargs.get('process_input', None) - addl_env=kwargs.get('addl_env', None) - check_exit_code=kwargs.get('check_exit_code', 0) - stdin=kwargs.get('stdin', subprocess.PIPE) - stdout=kwargs.get('stdout', subprocess.PIPE) - stderr=kwargs.get('stderr', subprocess.PIPE) - cmd=map(str,cmd) + process_input = kwargs.get('process_input', None) + addl_env = kwargs.get('addl_env', None) + check_exit_code = kwargs.get('check_exit_code', 0) + stdin = kwargs.get('stdin', subprocess.PIPE) + stdout = kwargs.get('stdout', subprocess.PIPE) + stderr = kwargs.get('stderr', subprocess.PIPE) + cmd = map(str, cmd) LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() @@ -151,7 +151,8 @@ def execute(*cmd, **kwargs): obj.stdin.close() if obj.returncode: LOG.debug(_("Result was %s") % obj.returncode) - if check_exit_code is not None and obj.returncode != check_exit_code: + if type(check_exit_code) == types.IntType \ + and obj.returncode != check_exit_code: (stdout, stderr) = result raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, diff --git a/nova/volume/driver.py b/nova/volume/driver.py index e9bdf162f..45cc800e7 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -112,7 +112,7 @@ class VolumeDriver(object): # If the volume isn't present, then don't attempt to delete return True - self._try_execute('sudo', 'lvremove', '-f',"%s/%s" % + self._try_execute('sudo', 'lvremove', '-f', "%s/%s" % (FLAGS.volume_group, volume['name'])) @@ -256,7 +256,7 @@ class ISCSIDriver(VolumeDriver): self._sync_exec('sudo', 'ietadm', '--op', 'new', "--tid=%s" % iscsi_target, '--params', - "Name=%s" % iscsi-name, + "Name=%s" % iscsi_name, check_exit_code=False) self._sync_exec('sudo', 'ietadm', '--op', 'new', "--tid=%s" % iscsi_target, -- cgit From 75f7a73735957d5ddf04c7c9a23decf1a6fa7f9f Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Wed, 9 Mar 2011 14:55:36 -0600 Subject: Added naming scheme comment --- nova/virt/xenapi_conn.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'nova') diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index b63a5f8c3..bfe290be3 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -49,6 +49,12 @@ reactor thread if the VM.get_by_name_label or VM.get_record calls block. address for the nova-volume host :target_port: iSCSI Target Port, 3260 Default :iqn_prefix: IQN Prefix, e.g. 'iqn.2010-10.org.openstack' + +**Variable Naming Scheme** + +- suffix "_ref" for opaque references +- suffix "_uuid" for UUIDs +- suffix "_rec" for record objects """ import sys -- cgit From 3e61bf9963d7e98e8152d2eacfc4461d8cda309c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 9 Mar 2011 21:43:35 +0000 Subject: remove the semaphore when there is no one waiting on it --- nova/tests/test_virt.py | 3 ++- nova/virt/libvirt_conn.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 113632a0c..56a271365 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -23,7 +23,6 @@ from xml.dom.minidom import parseString as xml_to_dom from nova import context from nova import db from nova import flags -from nova import log as logging from nova import test from nova import utils from nova.api.ec2 import cloud @@ -70,11 +69,13 @@ class CacheConcurrencyTestCase(test.TestCase): eventlet.sleep(0) try: self.assertFalse(done2.ready()) + self.assertTrue('fname' in conn._image_sems) finally: wait1.send() done1.wait() eventlet.sleep(0) self.assertTrue(done2.ready()) + self.assertFalse('fname' in conn._image_sems) def test_different_fname_concurrency(self): """Ensures that two different fname caches are concurrent""" diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index ecef7950a..69249ed57 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -539,6 +539,8 @@ class LibvirtConnection(object): with LibvirtConnection._image_sems[fname]: if not os.path.exists(base): fn(target=base, *args, **kwargs) + if not LibvirtConnection._image_sems[fname].locked(): + del LibvirtConnection._image_sems[fname] if cow: utils.execute('qemu-img create -f qcow2 -o ' -- cgit From e8554da80ac916f168461cb48078488700081c02 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 16:44:48 -0500 Subject: execvp: cleanup. --- nova/crypto.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/crypto.py b/nova/crypto.py index 717ea0041..2a8d4abca 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -105,7 +105,7 @@ def generate_key_pair(bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.join(tmpdir, 'temp') - utils.execute('ssh-keygen', '-q', '-b', '%d' % bits, '-N', '', + utils.execute('ssh-keygen', '-q', '-b', bits, '-N', '', '-f', keyfile) (out, err) = utils.execute('ssh-keygen', '-q', '-l', '-f', '%s.pub' % (keyfile)) @@ -147,9 +147,9 @@ def revoke_cert(project_id, file_name): os.chdir(ca_folder(project_id)) # NOTE(vish): potential race condition here utils.execute('openssl', 'ca', '-config', './openssl.cnf', '-revoke', - '%s' % file_name) + file_name) utils.execute('openssl', 'ca', '-gencrl', '-config', './openssl.cnf', - '-out', '%s' % FLAGS.crl_file) + '-out', FLAGS.crl_file) os.chdir(start) -- cgit From fb4785b85c1bef4179140cfb85ce01eca9fb5da5 Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Wed, 9 Mar 2011 21:46:27 +0000 Subject: fix the copyright notice in migration --- .../sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py index 514b92b81..eb3066894 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/010_add_os_type_to_instances.py @@ -1,8 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. +# Copyright 2010 OpenStack LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain -- cgit From 203e23ebebc73a98dc8e8497fd2b28d3a6bf01da Mon Sep 17 00:00:00 2001 From: Ken Pepple Date: Wed, 9 Mar 2011 14:13:52 -0800 Subject: initializing instance power state on launch to 0 (fixes EC2 API bug) --- nova/compute/api.py | 1 + 1 file changed, 1 insertion(+) (limited to 'nova') diff --git a/nova/compute/api.py b/nova/compute/api.py index a0bb2cf04..2358b562c 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -165,6 +165,7 @@ class API(base.Base): 'image_id': image_id, 'kernel_id': kernel_id or '', 'ramdisk_id': ramdisk_id or '', + 'state': "0", 'state_description': 'scheduling', 'user_id': context.user_id, 'project_id': context.project_id, -- cgit From 9822af58162dc520c4a17646a013560e422efcf9 Mon Sep 17 00:00:00 2001 From: Ken Pepple Date: Wed, 9 Mar 2011 14:54:57 -0800 Subject: maybe a int instead ? --- nova/compute/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/compute/api.py b/nova/compute/api.py index 2358b562c..5334acfcf 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -165,7 +165,7 @@ class API(base.Base): 'image_id': image_id, 'kernel_id': kernel_id or '', 'ramdisk_id': ramdisk_id or '', - 'state': "0", + 'state': 0, 'state_description': 'scheduling', 'user_id': context.user_id, 'project_id': context.project_id, -- cgit From 21937b48fcac81fa108f37f307b1b2e969bb7b4f Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Thu, 10 Mar 2011 00:01:15 +0000 Subject: Replace session.execute() calls performing raw UPDATE statements with SQLAlchemy code, with the exception of fixed_ip_disassociate_all_by_timeout() --- nova/db/sqlalchemy/api.py | 97 ++++++++++++++++++++++++++++++----------------- 1 file changed, 62 insertions(+), 35 deletions(-) (limited to 'nova') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 5e498fc6f..22c85106d 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -701,14 +701,18 @@ def instance_data_get_for_project(context, project_id): def instance_destroy(context, instance_id): session = get_session() with session.begin(): - session.execute('update instances set deleted=1,' - 'deleted_at=:at where id=:id', - {'id': instance_id, - 'at': datetime.datetime.utcnow()}) - session.execute('update security_group_instance_association ' - 'set deleted=1,deleted_at=:at where instance_id=:id', - {'id': instance_id, - 'at': datetime.datetime.utcnow()}) + session.query(models.Instance).\ + filter_by(id=instance_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': models.Instance.updated_at + 0}) + session.query(models.SecurityGroupInstanceAssociation).\ + filter_by(instance_id=instance_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': + (models.SecurityGroupInstanceAssociation. + updated_at + 0)}) @require_context @@ -950,9 +954,11 @@ def key_pair_destroy_all_by_user(context, user_id): authorize_user_context(context, user_id) session = get_session() with session.begin(): - # TODO(vish): do we have to use sql here? - session.execute('update key_pairs set deleted=1 where user_id=:id', - {'id': user_id}) + session.query(models.KeyPair).\ + filter_by(user_id=user_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': models.KeyPair.updated_at + 0}) @require_context @@ -1063,7 +1069,9 @@ def network_disassociate(context, network_id): @require_admin_context def network_disassociate_all(context): session = get_session() - session.execute('update networks set project_id=NULL') + session.query(models.Network).\ + update({'project_id': None, + 'updated_at': models.Network.updated_at + 0}) @require_context @@ -1433,15 +1441,17 @@ def volume_data_get_for_project(context, project_id): def volume_destroy(context, volume_id): session = get_session() with session.begin(): - # TODO(vish): do we have to use sql here? - session.execute('update volumes set deleted=1 where id=:id', - {'id': volume_id}) - session.execute('update export_devices set volume_id=NULL ' - 'where volume_id=:id', - {'id': volume_id}) - session.execute('update iscsi_targets set volume_id=NULL ' - 'where volume_id=:id', - {'id': volume_id}) + session.query(models.Volume).\ + filter_by(id=volume_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': models.Volume.updated_at + 0}) + session.query(models.ExportDevice).\ + filter_by(volume_id=volume_id).\ + update({'volume_id': None}) + session.query(models.IscsiTarget).\ + filter_by(volume_id=volume_id).\ + update({'volume_id': None}) @require_admin_context @@ -1661,17 +1671,26 @@ def security_group_create(context, values): def security_group_destroy(context, security_group_id): session = get_session() with session.begin(): - # TODO(vish): do we have to use sql here? - session.execute('update security_groups set deleted=1 where id=:id', - {'id': security_group_id}) - session.execute('update security_group_instance_association ' - 'set deleted=1,deleted_at=:at ' - 'where security_group_id=:id', - {'id': security_group_id, - 'at': datetime.datetime.utcnow()}) - session.execute('update security_group_rules set deleted=1 ' - 'where group_id=:id', - {'id': security_group_id}) + session.query(models.SecurityGroup).\ + filter_by(id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': + models.SecurityGroup.updated_at + 0}) + session.query(models.SecurityGroupInstanceAssociation).\ + filter_by(security_group_id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': + (models.SecurityGroupInstanceAssocation. + updated_at + 0)}) + session.query(models.SecurityGroupIngressRule).\ + filter_by(group_id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': + (models.SecurityGroupIngressRule. + updated_at + 0)}) @require_context @@ -1679,9 +1698,17 @@ def security_group_destroy_all(context, session=None): if not session: session = get_session() with session.begin(): - # TODO(vish): do we have to use sql here? - session.execute('update security_groups set deleted=1') - session.execute('update security_group_rules set deleted=1') + session.query(models.SecurityGroup).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': + models.SecurityGroup.updated_at + 0}) + session.query(models.SecurityGroupIngressRule).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': + (models.SecurityGroupIngressRule. + updated_at + 0)}) ################### -- cgit From f0bb6d9fc47b92d335c7d7fa238dfd43f0dbdf69 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Thu, 10 Mar 2011 13:30:52 +0900 Subject: fixed based on reviewer's comment. --- nova/compute/manager.py | 22 ++++++------- nova/db/sqlalchemy/api.py | 4 +-- nova/scheduler/driver.py | 10 +++--- nova/tests/test_compute.py | 24 +++++++------- nova/tests/test_scheduler.py | 5 +-- nova/tests/test_service.py | 77 +++++++++++++++++--------------------------- nova/tests/test_virt.py | 6 ++-- nova/tests/test_volume.py | 7 ++-- nova/virt/libvirt_conn.py | 22 ++++++++----- nova/volume/driver.py | 2 +- 10 files changed, 85 insertions(+), 94 deletions(-) (limited to 'nova') diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 3675cc92e..0cab10fc3 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -64,7 +64,7 @@ flags.DEFINE_integer('password_length', 12, flags.DEFINE_string('console_host', socket.gethostname(), 'Console proxy host to use to connect to instances on' 'this host.') -flags.DEFINE_string('live_migration_retry_count', 30, +flags.DEFINE_integer('live_migration_retry_count', 30, ("Retry count needed in live_migration." " sleep 1 sec for each count")) @@ -757,8 +757,9 @@ class ComputeManager(manager.Manager): dirpath = FLAGS.instances_path fd, tmp_file = tempfile.mkstemp(dir=dirpath) LOG.debug(_("Creating tmpfile %s to notify to other " - "compute node that they mounts same storage.") % tmp_file) - os.fdopen(fd, 'w+').close() + "compute nodes that they should mount " + "the same storage.") % tmp_file) + os.close(fd) return os.path.basename(tmp_file) @exception.wrap_exception @@ -812,7 +813,7 @@ class ComputeManager(manager.Manager): # Getting fixed ips fixed_ip = self.db.instance_get_fixed_address(context, instance_id) if not fixed_ip: - msg = _("%(instance_id)s(%(ec2_id)s) does'nt have fixed_ip") + msg = _("%(instance_id)s(%(ec2_id)s) does not have fixed_ip.") raise exception.NotFound(msg % locals()) # If any volume is mounted, prepare here. @@ -929,7 +930,7 @@ class ComputeManager(manager.Manager): floating_ip = self.db.instance_get_floating_address(ctxt, instance_id) if not floating_ip: - LOG.info(_('floating_ip is not found for %s'), i_name) + LOG.info(_('No floating_ip is found for %s.'), i_name) else: floating_ip_ref = self.db.floating_ip_get_by_address(ctxt, floating_ip) @@ -937,7 +938,7 @@ class ComputeManager(manager.Manager): floating_ip_ref['address'], {'host': dest}) except exception.NotFound: - LOG.info(_('Floating_ip is not found for %s'), i_name) + LOG.info(_('No floating_ip is found for %s.'), i_name) except: LOG.error(_("Live migration: Unexpected error:" "%s cannot inherit floating ip..") % i_name) @@ -945,12 +946,11 @@ class ComputeManager(manager.Manager): # Restore instance/volume state self.recover_live_migration(ctxt, instance_ref, dest) - LOG.info(_('Migrating %(i_name)s to %(dest)s finishes successfully.') + LOG.info(_('Migrating %(i_name)s to %(dest)s finished successfully.') % locals()) - LOG.info(_("The below error is normally occurs. " - "Just check if instance is successfully migrated.\n" - "libvir: QEMU error : Domain not found: no domain " - "with matching name..")) + LOG.info(_("You may see the error \"libvirt: QEMU error: " + "Domain not found: no domain with matching name.\" " + "This error can be safely ignored.")) def recover_live_migration(self, ctxt, instance_ref, host=None): """Recovers Instance/volume state from migrating -> running. diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 8ea5062ae..f44ca0fa3 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -192,8 +192,8 @@ def service_get_all_compute_by_host(context, host): all() if not result: - raise exception.NotFound(_("%s does not exist or not " - "compute node.") % host) + raise exception.NotFound(_("%s does not exist or is not " + "a compute node.") % host) return result diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 791f9000d..ed3dfe1c0 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -226,7 +226,6 @@ class Scheduler(object): "args": {'cpu_info': oservice_ref['cpu_info']}}) except rpc.RemoteError: - ec2_id = instance_ref['hostname'] src = instance_ref['host'] logging.exception(_("host %(dest)s is not compatible with " "original host %(src)s.") % locals()) @@ -259,9 +258,10 @@ class Scheduler(object): mem_avail = mem_total - mem_used mem_inst = instance_ref['memory_mb'] if mem_avail <= mem_inst: - raise exception.NotEmpty(_("%(ec2_id)s is not capable to " - "migrate %(dest)s (host:%(mem_avail)s " - " <= instance:%(mem_inst)s)") + raise exception.NotEmpty(_("Unable to migrate %(ec2_id)s " + "to destination: %(dest)s " + "(host:%(mem_avail)s " + "<= instance:%(mem_inst)s)") % locals()) def mounted_on_same_shared_storage(self, context, instance_ref, dest): @@ -292,7 +292,7 @@ class Scheduler(object): except rpc.RemoteError: ipath = FLAGS.instances_path - logging.error(_("Cannot comfirm tmpfile at %(ipath)s is on " + logging.error(_("Cannot confirm tmpfile at %(ipath)s is on " "same shared storage between %(src)s " "and %(dest)s.") % locals()) raise diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 85c2c948b..71899ba9e 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -89,14 +89,14 @@ class ComputeTestCase(test.TestCase): Use this when any testcase executed later than test_run_terminate """ vol1 = models.Volume() - vol1.__setitem__('id', 1) + vol1['id'] = 1 vol2 = models.Volume() - vol2.__setitem__('id', 2) + vol2['id'] = 2 instance_ref = models.Instance() - instance_ref.__setitem__('id', 1) - instance_ref.__setitem__('volumes', [vol1, vol2]) - instance_ref.__setitem__('hostname', 'i-00000001') - instance_ref.__setitem__('host', 'dummy') + instance_ref['id'] = 1 + instance_ref['volumes'] = [vol1, vol2] + instance_ref['hostname'] = 'i-00000001' + instance_ref['host'] = 'dummy' return instance_ref def test_create_instance_defaults_display_name(self): @@ -114,9 +114,9 @@ class ComputeTestCase(test.TestCase): """Make sure create associates security groups""" group = self._create_group() instance_ref = models.Instance() - instance_ref.__setitem__('id', 1) - instance_ref.__setitem__('volumes', [{'id': 1}, {'id': 2}]) - instance_ref.__setitem__('hostname', 'i-00000001') + instance_ref['id'] = 1 + instance_ref['volumes'] = [{'id': 1}, {'id': 2}] + instance_ref['hostname'] = 'i-00000001' return instance_ref def test_create_instance_defaults_display_name(self): @@ -390,7 +390,7 @@ class ComputeTestCase(test.TestCase): def test_pre_live_migration_instance_has_no_volume(self): """Confirm log meg when instance doesn't mount any volumes.""" i_ref = self._get_dummy_instance() - i_ref.__setitem__('volumes', []) + i_ref['volumes'] = [] c = context.get_admin_context() self._setup_other_managers() @@ -501,7 +501,7 @@ class ComputeTestCase(test.TestCase): def test_live_migration_dest_raises_exception_no_volume(self): """Same as above test(input pattern is different) """ i_ref = self._get_dummy_instance() - i_ref.__setitem__('volumes', []) + i_ref['volumes'] = [] c = context.get_admin_context() topic = db.queue_get_for(c, FLAGS.compute_topic, i_ref['host']) @@ -526,7 +526,7 @@ class ComputeTestCase(test.TestCase): def test_live_migration_works_correctly_no_volume(self): """Confirm live_migration() works as expected correctly.""" i_ref = self._get_dummy_instance() - i_ref.__setitem__('volumes', []) + i_ref['volumes'] = [] c = context.get_admin_context() topic = db.queue_get_for(c, FLAGS.compute_topic, i_ref['host']) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 711b66af7..8ac02c5a4 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -119,7 +119,8 @@ class SchedulerTestCase(test.TestCase): try: scheduler.show_host_resources(ctxt, dest) except exception.NotFound, e: - c1 = (0 <= e.message.find('does not exist or not compute node')) + c1 = (e.message.find(_("does not exist or is not a " + "compute node.")) >= 0) self.assertTrue(c1) def _dic_is_equal(self, dic1, dic2, keys=None): @@ -786,7 +787,7 @@ class SimpleDriverTestCase(test.TestCase): i_ref, 'somewhere') except exception.NotEmpty, e: - c = (e.message.find('is not capable to migrate') >= 0) + c = (e.message.find('Unable to migrate') >= 0) self.assertTrue(c) db.instance_destroy(self.context, instance_id) diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index d17f6a22a..666c4a11d 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -42,24 +42,6 @@ class FakeManager(manager.Manager): def test_method(self): return 'manager' -# temporary variable to store host/binary/self.mox -# from each method to fake class. -global_host = None -global_binary = None -global_mox = None - - -class FakeComputeManager(compute_manager.ComputeManager): - """Fake computemanager manager for tests""" - - def __init__(self, compute_driver=None, *args, **kwargs): - global ghost, gbinary, gmox - self.update_available_resource(mox.IgnoreArg()) - gmox.ReplayAll() - super(FakeComputeManager, self).__init__(compute_driver, - *args, - **kwargs) - class ExtendedService(service.Service): def test_method(self): @@ -275,37 +257,38 @@ class ServiceTestCase(test.TestCase): """Confirm compute updates their record of compute-service table.""" host = 'foo' binary = 'nova-compute' - topic = 'compute1' - service_create = {'host': host, - 'binary': binary, - 'topic': topic, - 'report_count': 0, - 'availability_zone': 'nova'} - service_ref = {'host': host, - 'binary': binary, - 'topic': topic, - 'report_count': 0, - 'availability_zone': 'nova', - 'id': 1} - - service.db.service_get_by_args(mox.IgnoreArg(), - host, - binary).AndRaise(exception.NotFound()) - service.db.service_create(mox.IgnoreArg(), - service_create).AndReturn(service_ref) - self.mox.StubOutWithMock(compute_manager.ComputeManager, - 'update_available_resource') - - global ghost, gbinary, gmox - ghost = host - gbinary = binary - gmox = self.mox - + topic = 'compute' + + # Any mocks are not working without UnsetStubs() here. + self.mox.UnsetStubs() + ctxt = context.get_admin_context() + service_ref = db.service_create(ctxt, {'host': host, + 'binary': binary, + 'topic': topic}) serv = service.Service(host, binary, topic, - 'nova.tests.test_service.FakeComputeManager') - # ReplayAll has been executed FakeComputeManager.__init__() - #self.mox.ReplayAll() + 'nova.compute.manager.ComputeManager') + + # This testcase want to test calling update_available_resource. + # No need to call periodic call, then below variable must be set 0. + serv.report_interval = 0 + serv.periodic_interval = 0 + + # Creating mocks + self.mox.StubOutWithMock(service.rpc.Connection, 'instance') + service.rpc.Connection.instance(new=mox.IgnoreArg()) + service.rpc.Connection.instance(new=mox.IgnoreArg()) + self.mox.StubOutWithMock(serv.manager.driver, + 'update_available_resource') + serv.manager.driver.update_available_resource(mox.IgnoreArg(), host) + + # Just doing start()-stop(), not confirm new db record is created, + # because update_available_resource() works only in libvirt environment. + # This testcase confirms update_available_resource() is called. + # Otherwise, mox complains. + self.mox.ReplayAll() serv.start() serv.stop() + + db.service_destroy(ctxt, service_ref['id']) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 7ea8c0fb5..ee41ae732 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -283,7 +283,7 @@ class LibvirtConnTestCase(test.TestCase): self.assertEquals(uri, testuri) db.instance_destroy(user_context, instance_ref['id']) - def tes1t_update_available_resource_works_correctly(self): + def test_update_available_resource_works_correctly(self): """Confirm compute_node table is updated successfully.""" org_path = FLAGS.instances_path = '' FLAGS.instances_path = '.' @@ -314,7 +314,7 @@ class LibvirtConnTestCase(test.TestCase): compute_node = service_ref['compute_node'][0] if sys.platform.upper() == 'LINUX2': - self.assertTrue(compute_node['vcpus'] > 0) + self.assertTrue(compute_node['vcpus'] >= 0) self.assertTrue(compute_node['memory_mb'] > 0) self.assertTrue(compute_node['local_gb'] > 0) self.assertTrue(compute_node['vcpus_used'] == 0) @@ -323,7 +323,7 @@ class LibvirtConnTestCase(test.TestCase): self.assertTrue(len(compute_node['hypervisor_type']) > 0) self.assertTrue(compute_node['hypervisor_version'] > 0) else: - self.assertTrue(compute_node['vcpus'] > 0) + self.assertTrue(compute_node['vcpus'] >= 0) self.assertTrue(compute_node['memory_mb'] == 0) self.assertTrue(compute_node['local_gb'] > 0) self.assertTrue(compute_node['vcpus_used'] == 0) diff --git a/nova/tests/test_volume.py b/nova/tests/test_volume.py index e8b4ceee8..d88e363da 100644 --- a/nova/tests/test_volume.py +++ b/nova/tests/test_volume.py @@ -284,9 +284,10 @@ class AOETestCase(DriverTestCase): self.volume.check_for_export(self.context, self.instance_id) except exception.ProcessExecutionError, e: volume_id = volume_id_list[0] - msg = _("""Cannot confirm exported volume id:%(volume_id)s.""" - """vblade process for e%(shelf_id)s.%(blade_id)s """ - """isn't running.""") % locals() + msg = _("Cannot confirm exported volume id:%(volume_id)s. " + "vblade process for e%(shelf_id)s.%(blade_id)s " + "isn't running.") % locals() + msg_is_match = (0 <= e.message.find(msg)) self.assertTrue(msg_is_match) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 938719a7c..43a9dc4e7 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -860,7 +860,14 @@ class LibvirtConnection(object): """ - return multiprocessing.cpu_count() + # On certain platforms, this will raise a NotImplementedError. + try: + return multiprocessing.cpu_count() + except NotImplementedError: + LOG.warn(_("Cannot get the number of cpu, because this " + "function is not implemented for this platform. " + "This error can be safely ignored for now.")) + return 0 def get_memory_mb_total(self): """Get the total memory size(MB) of physical computer. @@ -1042,9 +1049,9 @@ class LibvirtConnection(object): try: service_ref = db.service_get_all_compute_by_host(ctxt, host)[0] except exception.NotFound: - msg = _(("""Cannot update compute manager specific info,""" - """ Because no service record found.""")) - raise exception.Invalid(msg) + raise exception.Invalid(_("Cannot update compute manager " + "specific info, because no service " + "record was found.")) # Updating host information dic = {'vcpus': self.get_vcpu_total(), @@ -1059,11 +1066,11 @@ class LibvirtConnection(object): compute_node_ref = service_ref['compute_node'] if not compute_node_ref: - LOG.info(_('Compute_service record is created for %s ') % host) + LOG.info(_('Compute_service record created for %s ') % host) dic['service_id'] = service_ref['id'] db.compute_node_create(ctxt, dic) else: - LOG.info(_('Compute_service record is updated for %s ') % host) + LOG.info(_('Compute_service record updated for %s ') % host) db.compute_node_update(ctxt, compute_node_ref[0]['id'], dic) def compare_cpu(self, cpu_info): @@ -1081,8 +1088,7 @@ class LibvirtConnection(object): """ - LOG.info(_('Checking cpu_info: instance was launched this cpu.\n%s') - % cpu_info) + LOG.info(_('Instance launched has CPU info:\n%s') % cpu_info) dic = utils.loads(cpu_info) xml = str(Template(self.cpuinfo_xml, searchList=dic)) LOG.info(_('to xml...\n:%s ' % xml)) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index a902da6ac..31a6a02ee 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -234,7 +234,7 @@ class AOEDriver(VolumeDriver): break if not exported: # Instance will be terminated in this case. - desc = _("Cannot confirm exported volume id:%(volume_id)s." + desc = _("Cannot confirm exported volume id:%(volume_id)s. " "vblade process for e%(shelf_id)s.%(blade_id)s " "isn't running.") % locals() raise exception.ProcessExecutionError(out, _err, cmd=cmd, -- cgit From a83b4879f38d11634d405d0efe977d482abdc344 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Thu, 10 Mar 2011 05:02:24 +0000 Subject: minor fixes from review --- nova/image/glance.py | 2 +- nova/image/s3.py | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) (limited to 'nova') diff --git a/nova/image/glance.py b/nova/image/glance.py index fb383f5e6..15fca69b8 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -74,7 +74,7 @@ class GlanceImageService(service.BaseImageService): if name == cantidate.get('name'): image = cantidate break - if image == None: + if image is None: raise exception.NotFound return image diff --git a/nova/image/s3.py b/nova/image/s3.py index bf104c29a..bbc54c263 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -77,17 +77,17 @@ class S3ImageService(service.BaseImageService): # FIXME(vish): detail doesn't filter so we do it manually return self._filter(context, images) - @staticmethod - def _is_visible(context, image): + @classmethod + def _is_visible(cls, context, image): return (context.is_admin or context.project_id == image['properties']['owner_id'] or image['properties']['is_public'] == 'True') - @staticmethod - def _filter(context, images): + @classmethod + def _filter(cls, context, images): filtered = [] for image in images: - if not S3ImageService._is_visible(context, image): + if not cls._is_visible(context, image): continue filtered.append(image) return filtered @@ -148,7 +148,7 @@ class S3ImageService(service.BaseImageService): image_format = 'aki' image_type = 'kernel' kernel_id = None - except: + except Exception: kernel_id = None try: @@ -157,12 +157,12 @@ class S3ImageService(service.BaseImageService): image_format = 'ari' image_type = 'ramdisk' ramdisk_id = None - except: + except Exception: ramdisk_id = None try: arch = manifest.find("machine_configuration/architecture").text - except: + except Exception: arch = 'x86_64' properties = metadata['properties'] @@ -235,7 +235,7 @@ class S3ImageService(service.BaseImageService): @staticmethod def _decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, - cloud_private_key, decrypted_filename): + cloud_private_key, decrypted_filename): key, err = utils.execute( 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, process_input=encrypted_key, -- cgit From 90f38451e5df4f0ca862401cf898f01ffede6174 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 10 Mar 2011 00:26:25 -0500 Subject: add tests to verify the serialization of adminPass in server creation response --- nova/tests/api/openstack/test_servers.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 2fc28fe67..0561ad499 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -19,6 +19,7 @@ import base64 import datetime import json import unittest +from xml.dom import minidom import stubout import webob @@ -908,6 +909,7 @@ class TestServerInstanceCreation(test.TestCase): def _get_create_request_xml(self, body_dict): req = webob.Request.blank('/v1.0/servers') req.content_type = 'application/xml' + req.accept = 'application/xml' req.method = 'POST' req.body = self._format_xml_request_body(body_dict) return req @@ -1034,6 +1036,23 @@ class TestServerInstanceCreation(test.TestCase): self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, [(path, contents)]) + def test_create_instance_admin_pass_json(self): + request, response, dummy = \ + self._create_instance_with_personality_json(None) + self.assertEquals(response.status_int, 200) + response = json.loads(response.body) + self.assertTrue('adminPass' in response['server']) + self.assertTrue(response['server']['adminPass'].startswith('fake')) + + def test_create_instance_admin_pass_xml(self): + request, response, dummy = \ + self._create_instance_with_personality_xml(None) + self.assertEquals(response.status_int, 200) + dom = minidom.parseString(response.body) + server = dom.childNodes[0] + self.assertEquals(server.nodeName, 'server') + self.assertTrue(server.getAttribute('adminPass').startswith('fake')) + if __name__ == "__main__": unittest.main() -- cgit From b75ab789194f1ced801b1d68ae8cc54051716414 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Thu, 10 Mar 2011 15:16:03 +0900 Subject: fix pep8 check --- nova/tests/test_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index 666c4a11d..393f9d20b 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -284,9 +284,9 @@ class ServiceTestCase(test.TestCase): serv.manager.driver.update_available_resource(mox.IgnoreArg(), host) # Just doing start()-stop(), not confirm new db record is created, - # because update_available_resource() works only in libvirt environment. - # This testcase confirms update_available_resource() is called. - # Otherwise, mox complains. + # because update_available_resource() works only in + # libvirt environment. This testcase confirms + # update_available_resource() is called. Otherwise, mox complains. self.mox.ReplayAll() serv.start() serv.stop() -- cgit From 1fa41c5c621f3190c8c2b1c3d885c95b6b627b23 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 09:52:19 +0100 Subject: s/s.getuid()/os.getuid()/ --- nova/virt/libvirt_conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index e25e4af4f..44b07213a 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -463,7 +463,7 @@ class LibvirtConnection(object): console_log = os.path.join(FLAGS.instances_path, instance['name'], 'console.log') - utils.execute('sudo', 'chown', s.getuid(), console_log) + utils.execute('sudo', 'chown', os.getuid(), console_log) if FLAGS.libvirt_type == 'xen': # Xen is special -- cgit From 9e77a0c6f6b43494e0eb87a16f33cd566f0746d2 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 09:55:45 +0100 Subject: Split dnsmasq and radvd commands into their respective argv's. --- nova/network/linux_net.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 228a4d9ea..9fd6c82de 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -424,30 +424,30 @@ def _remove_rule(chain, *cmd): def _dnsmasq_cmd(net): """Builds dnsmasq command""" - cmd = ['sudo -E dnsmasq', - ' --strict-order', - ' --bind-interfaces', - ' --conf-file=', - ' --domain=%s' % FLAGS.dhcp_domain, - ' --pid-file=%s' % _dhcp_file(net['bridge'], 'pid'), - ' --listen-address=%s' % net['gateway'], - ' --except-interface=lo', - ' --dhcp-range=%s,static,120s' % net['dhcp_start'], - ' --dhcp-hostsfile=%s' % _dhcp_file(net['bridge'], 'conf'), - ' --dhcp-script=%s' % FLAGS.dhcpbridge, - ' --leasefile-ro'] + cmd = ['sudo', '-E', 'dnsmasq', + '--strict-order', + '--bind-interfaces', + '--conf-file=', + '--domain=%s' % FLAGS.dhcp_domain, + '--pid-file=%s' % _dhcp_file(net['bridge'], 'pid'), + '--listen-address=%s' % net['gateway'], + '--except-interface=lo', + '--dhcp-range=%s,static,120s' % net['dhcp_start'], + '--dhcp-hostsfile=%s' % _dhcp_file(net['bridge'], 'conf'), + '--dhcp-script=%s' % FLAGS.dhcpbridge, + '--leasefile-ro'] if FLAGS.dns_server: - cmd.append(' -h -R --server=%s' % FLAGS.dns_server) - return ''.join(cmd) + cmd += ['-h', '-R', '--server=%s' % FLAGS.dns_server] + return cmd def _ra_cmd(net): """Builds radvd command""" - cmd = ['sudo -E radvd', -# ' -u nobody', - ' -C %s' % _ra_file(net['bridge'], 'conf'), - ' -p %s' % _ra_file(net['bridge'], 'pid')] - return ''.join(cmd) + cmd = ['sudo', '-E', 'radvd', +# '-u', 'nobody', + '-C', '%s' % _ra_file(net['bridge'], 'conf'), + '-p', '%s' % _ra_file(net['bridge'], 'pid')] + return cmd def _stop_dnsmasq(network): -- cgit From e575f5ddd46055f2e491606052493b6d648506f6 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 10:16:07 +0100 Subject: Pass argv of dnsmasq and radvd to execute as individual args, not as a list. --- nova/network/linux_net.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 9fd6c82de..e64c052f9 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -330,7 +330,7 @@ def update_dhcp(context, network_id): env = {'FLAGFILE': FLAGS.dhcpbridge_flagfile, 'DNSMASQ_INTERFACE': network_ref['bridge']} command = _dnsmasq_cmd(network_ref) - _execute(command, addl_env=env) + _execute(*command, addl_env=env) def update_ra(context, network_id): @@ -370,7 +370,7 @@ interface %s else: LOG.debug(_("Pid %d is stale, relaunching radvd"), pid) command = _ra_cmd(network_ref) - _execute(command) + _execute(*command) db.network_update(context, network_id, {"ra_server": utils.get_my_linklocal(network_ref['bridge'])}) -- cgit From 6601d52bfa501ac1ae266647be19fac2f6792efc Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 11:35:42 +0100 Subject: Make nova.image.s3 catch up with the new execute syntax. --- nova/image/s3.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) (limited to 'nova') diff --git a/nova/image/s3.py b/nova/image/s3.py index bbc54c263..85a2c651c 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -236,25 +236,32 @@ class S3ImageService(service.BaseImageService): @staticmethod def _decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, cloud_private_key, decrypted_filename): - key, err = utils.execute( - 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, - process_input=encrypted_key, - check_exit_code=False) + key, err = utils.execute('openssl', + 'rsautl', + '-decrypt', + '-inkey', '%s' % cloud_private_key, + process_input=encrypted_key, + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt private key: %s") % err) - iv, err = utils.execute( - 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, - process_input=encrypted_iv, - check_exit_code=False) + iv, err = utils.execute('openssl', + 'rsautl', + '-decrypt', + '-inkey', '%s' % cloud_private_key, + process_input=encrypted_iv, + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt initialization " "vector: %s") % err) - _out, err = utils.execute( - 'openssl enc -d -aes-128-cbc -in %s -K %s -iv %s -out %s' - % (encrypted_filename, key, iv, decrypted_filename), - check_exit_code=False) + _out, err = utils.execute('openssl', 'enc', + '-d', '-aes-128-cbc', + '-in', '%s' % (encrypted_filename,), + '-K', '%s' % (key,), + '-iv', '%s' % (iv,), + '-out', '%s' % (decrypted_filename,), + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt image file " "%(image_file)s: %(err)s") % -- cgit From bd3411f88532619b760aa8f51379db2f9c1cf5d0 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 11:59:50 +0100 Subject: More execvp fallout --- nova/objectstore/image.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) (limited to 'nova') diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index 8013cbd9c..c90b5b54b 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -253,25 +253,34 @@ class Image(object): @staticmethod def decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, cloud_private_key, decrypted_filename): - key, err = utils.execute( - 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, - process_input=encrypted_key, - check_exit_code=False) + key, err = utils.execute('openssl', + 'rsautl', + '-decrypt', + '-inkey', '%s' % cloud_private_key, + process_input=encrypted_key, + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt private key: %s") % err) - iv, err = utils.execute( - 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, - process_input=encrypted_iv, - check_exit_code=False) + iv, err = utils.execute('openssl', + 'rsautl', + '-decrypt', + '-inkey', '%s' % cloud_private_key, + process_input=encrypted_iv, + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt initialization " "vector: %s") % err) - _out, err = utils.execute( - 'openssl enc -d -aes-128-cbc -in %s -K %s -iv %s -out %s' - % (encrypted_filename, key, iv, decrypted_filename), - check_exit_code=False) + _out, err = utils.execute('openssl', + 'enc', + '-d', + '-aes-128-cbc', + '-in', '%s' % (encrypted_filename,), + '-K', '%s' % (key,), + '-iv', '%s' % (iv,), + '-out', '%s' % (decrypted_filename,), + check_exit_code=False) if err: raise exception.Error(_("Failed to decrypt image file " "%(image_file)s: %(err)s") % -- cgit From e76aad24ce8a9b1b7de1b2f874c22c9995f3071f Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 14:30:17 +0100 Subject: Only include ramdisk and kernel id if they are actually set. --- nova/api/ec2/cloud.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'nova') diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index b1917e9ea..1d2254225 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -147,8 +147,6 @@ class CloudController(object): instance_ref['id']) ec2_id = ec2utils.id_to_ec2_id(instance_ref['id']) image_ec2_id = self._image_ec2_id(instance_ref['image_id'], 'machine') - k_ec2_id = self._image_ec2_id(instance_ref['kernel_id'], 'kernel') - r_ec2_id = self._image_ec2_id(instance_ref['ramdisk_id'], 'ramdisk') data = { 'user-data': base64.b64decode(instance_ref['user_data']), 'meta-data': { @@ -167,8 +165,6 @@ class CloudController(object): 'instance-type': instance_ref['instance_type'], 'local-hostname': hostname, 'local-ipv4': address, - 'kernel-id': k_ec2_id, - 'ramdisk-id': r_ec2_id, 'placement': {'availability-zone': availability_zone}, 'public-hostname': hostname, 'public-ipv4': floating_ip or '', @@ -176,6 +172,13 @@ class CloudController(object): 'reservation-id': instance_ref['reservation_id'], 'security-groups': '', 'mpi': mpi}} + + for image_type in ['kernel', 'ramdisk']: + if '%s_id' % image_type in instance_ref: + ec2_id = self._image_ec2_id(instance_ref['%s_id' % image_type], + image_type) + data['meta-data']['%s-id' % image_type] = ec2_id + if False: # TODO(vish): store ancestor ids data['ancestor-ami-ids'] = [] if False: # TODO(vish): store product codes -- cgit From b64cf7352a24d8ced69aa408f7ceadd9da71da14 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 15:47:09 +0100 Subject: One more thing.. --- nova/network/linux_net.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index e64c052f9..c0bd76adf 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -216,7 +216,7 @@ def ensure_bridge(bridge, interface, net_attrs=None): _execute('sudo', 'brctl', 'setfd', bridge, 0) # _execute("sudo brctl setageing %s 10" % bridge) _execute('sudo', 'brctl', 'stp', bridge, 'off') - _execute('sudo', 'ip', 'link', 'set', bridge, up) + _execute('sudo', 'ip', 'link', 'set', bridge, 'up') if net_attrs: # NOTE(vish): The ip for dnsmasq has to be the first address on the # bridge for it to respond to reqests properly -- cgit From b38af111532717cbe9f4bef1d3c3d58e7082c8b9 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 16:25:18 +0100 Subject: Another little detail.. --- nova/virt/disk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/virt/disk.py b/nova/virt/disk.py index a54cda003..5d499c42c 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -51,7 +51,7 @@ def extend(image, size): return utils.execute('truncate', '-s', size, image) # NOTE(vish): attempts to resize filesystem - utils.execute('e2fsck', '-fp', mage, check_exit_code=False) + utils.execute('e2fsck', '-fp', image, check_exit_code=False) utils.execute('resize2fs', image, check_exit_code=False) -- cgit From 801212a0ff04ddc33719d17b8c8ca847db5b1228 Mon Sep 17 00:00:00 2001 From: Cory Wright Date: Thu, 10 Mar 2011 15:47:55 +0000 Subject: Use a FLAGS.default_os_type if available --- nova/virt/xenapi/vm_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'nova') diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index a1b85284f..8dd246178 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -41,9 +41,11 @@ from nova.virt.xenapi import HelperBase from nova.virt.xenapi.volume_utils import StorageError -FLAGS = flags.FLAGS LOG = logging.getLogger("nova.virt.xenapi.vm_utils") +FLAGS = flags.FLAGS +flags.DEFINE_string('default_os_type', 'linux', 'Default OS type') + XENAPI_POWER_STATE = { 'Halted': power_state.SHUTDOWN, 'Running': power_state.RUNNING, @@ -347,7 +349,7 @@ class VMHelper(HelperBase): logging.debug(_("Asking xapi to upload %(vdi_uuids)s as" " ID %(image_id)s") % locals()) - os_type = instance.os_type and instance.os_type or 'linux' + os_type = instance.os_type or FLAGS.default_os_type params = {'vdi_uuids': vdi_uuids, 'image_id': image_id, -- cgit From b361153a160ba1d61ed1d52de419cd27a8b4feda Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Thu, 10 Mar 2011 16:42:13 +0000 Subject: Correct a misspelling --- nova/db/sqlalchemy/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 22c85106d..2b60ab7d5 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -1682,7 +1682,7 @@ def security_group_destroy(context, security_group_id): update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), 'updated_at': - (models.SecurityGroupInstanceAssocation. + (models.SecurityGroupInstanceAssociation. updated_at + 0)}) session.query(models.SecurityGroupIngressRule).\ filter_by(group_id=security_group_id).\ -- cgit From 4ead485ab69ee1e92635857ba73133a9e1d3bbcb Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Thu, 10 Mar 2011 12:06:09 -0600 Subject: Cleaned up vmops --- nova/virt/xenapi/vm_utils.py | 26 +++++------ nova/virt/xenapi/vmops.py | 104 ++++++++++++++++++++----------------------- 2 files changed, 61 insertions(+), 69 deletions(-) (limited to 'nova') diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index ce081a2d6..4ad820bcd 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -90,7 +90,7 @@ class VMHelper(HelperBase): get_instance_type(instance.instance_type) mem = str(long(instance_type['memory_mb']) * 1024 * 1024) vcpus = str(instance_type['vcpus']) - rec = { + vm_rec = { 'name_label': instance.name, 'name_description': '', 'is_a_template': False, @@ -122,23 +122,23 @@ class VMHelper(HelperBase): #Complete VM configuration record according to the image type #non-raw/raw with PV kernel/raw in HVM mode if instance.kernel_id: - rec['PV_bootloader'] = '' - rec['PV_kernel'] = kernel - rec['PV_ramdisk'] = ramdisk - rec['PV_args'] = 'root=/dev/xvda1' - rec['PV_bootloader_args'] = '' - rec['PV_legacy_args'] = '' + vm_rec['PV_bootloader'] = '' + vm_rec['PV_kernel'] = kernel + vm_rec['PV_ramdisk'] = ramdisk + vm_rec['PV_args'] = 'root=/dev/xvda1' + vm_rec['PV_bootloader_args'] = '' + vm_rec['PV_legacy_args'] = '' else: if pv_kernel: - rec['PV_args'] = 'noninteractive' - rec['PV_bootloader'] = 'pygrub' + vm_rec['PV_args'] = 'noninteractive' + vm_rec['PV_bootloader'] = 'pygrub' else: - rec['HVM_boot_policy'] = 'BIOS order' - rec['HVM_boot_params'] = {'order': 'dc'} - rec['platform'] = {'acpi': 'true', 'apic': 'true', + vm_rec['HVM_boot_policy'] = 'BIOS order' + vm_rec['HVM_boot_params'] = {'order': 'dc'} + vm_rec['platform'] = {'acpi': 'true', 'apic': 'true', 'pae': 'true', 'viridian': 'true'} LOG.debug(_('Created VM %s...'), instance.name) - vm_ref = session.call_xenapi('VM.create', rec) + vm_ref = session.call_xenapi('VM.create', vm_rec) instance_name = instance.name LOG.debug(_('Created VM %(instance_name)s as %(vm_ref)s.') % locals()) return vm_ref diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 562ecd4d5..5375df5b4 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -56,10 +56,10 @@ class VMOps(object): def list_instances(self): """List VM instances""" vms = [] - for vm in self._session.get_xenapi().VM.get_all(): - rec = self._session.get_xenapi().VM.get_record(vm) - if not rec["is_a_template"] and not rec["is_control_domain"]: - vms.append(rec["name_label"]) + for vm_ref in self._session.get_xenapi().VM.get_all(): + vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) + if not vm_rec["is_a_template"] and not vm_rec["is_control_domain"]: + vms.append(vm_rec["name_label"]) return vms def _start(self, instance, vm_ref=None): @@ -371,8 +371,8 @@ class VMOps(object): def reboot(self, instance): """Reboot VM instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.clean_reboot', vm) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.clean_reboot', vm_ref) self._session.wait_for_task(task, instance.id) def set_admin_password(self, instance, new_pass): @@ -571,26 +571,27 @@ class VMOps(object): def pause(self, instance, callback): """Pause VM instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.pause', vm) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.pause', vm_ref) self._wait_with_callback(instance.id, task, callback) def unpause(self, instance, callback): """Unpause VM instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.unpause', vm) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.unpause', vm_ref) self._wait_with_callback(instance.id, task, callback) def suspend(self, instance, callback): """suspend the specified instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.suspend', vm) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.suspend', vm_ref) self._wait_with_callback(instance.id, task, callback) def resume(self, instance, callback): """resume the specified instance""" - vm = self._get_vm_opaque_ref(instance) - task = self._session.call_xenapi('Async.VM.resume', vm, False, True) + vm_ref = self._get_vm_opaque_ref(instance) + task = self._session.call_xenapi('Async.VM.resume', vm_ref, False, + True) self._wait_with_callback(instance.id, task, callback) def rescue(self, instance, callback): @@ -605,22 +606,18 @@ class VMOps(object): raise RuntimeError(_( "Instance is already in Rescue Mode: %s" % instance.name)) - vm = self._get_vm_opaque_ref(instance) - self._shutdown(instance, vm) - self._acquire_bootlock(vm) + vm_ref = self._get_vm_opaque_ref(instance) + self._shutdown(instance, vm_ref) + self._acquire_bootlock(vm_ref) instance._rescue = True self.spawn(instance) - rescue_vm = self._get_vm_opaque_ref(instance) + rescue_vm_ref = self._get_vm_opaque_ref(instance) - vbd = self._session.get_xenapi().VM.get_VBDs(vm)[0] + vbd = self._session.get_xenapi().VM.get_VBDs(vm_ref)[0] vdi_ref = self._session.get_xenapi().VBD.get_record(vbd)["VDI"] - vbd_ref = VMHelper.create_vbd( - self._session, - rescue_vm, - vdi_ref, - 1, - False) + vbd_ref = VMHelper.create_vbd(self._session, rescue_vm_ref, vdi_ref, + 1, False) self._session.call_xenapi("Async.VBD.plug", vbd_ref) @@ -637,7 +634,7 @@ class VMOps(object): raise exception.NotFound(_( "Instance is not in Rescue Mode: %s" % instance.name)) - original_vm = self._get_vm_opaque_ref(instance) + original_vm_ref = self._get_vm_opaque_ref(instance) vbds = self._session.get_xenapi().VM.get_VBDs(rescue_vm) instance._rescue = False @@ -662,20 +659,20 @@ class VMOps(object): task2 = self._session.call_xenapi('Async.VM.destroy', rescue_vm) self._session.wait_for_task(task2, instance.id) - self._release_bootlock(original_vm) - self._start(instance, original_vm) + self._release_bootlock(original_vm_ref) + self._start(instance, original_vm_ref) def get_info(self, instance): """Return data about VM instance""" - vm = self._get_vm_opaque_ref(instance) - rec = self._session.get_xenapi().VM.get_record(vm) - return VMHelper.compile_info(rec) + vm_ref = self._get_vm_opaque_ref(instance) + vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) + return VMHelper.compile_info(vm_rec) def get_diagnostics(self, instance): """Return data about VM diagnostics""" - vm = self._get_vm_opaque_ref(instance) - rec = self._session.get_xenapi().VM.get_record(vm) - return VMHelper.compile_diagnostics(self._session, rec) + vm_ref = self._get_vm_opaque_ref(instance) + vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) + return VMHelper.compile_diagnostics(self._session, vm_rec) def get_console_output(self, instance): """Return snapshot of console""" @@ -698,9 +695,9 @@ class VMOps(object): # at this stage even though they aren't implemented because these will # be needed for multi-nic and there was no sense writing it for single # network/single IP and then having to turn around and re-write it - vm_opaque_ref = self._get_vm_opaque_ref(instance.id) + vm_ref = self._get_vm_opaque_ref(instance.id) logging.debug(_("injecting network info to xenstore for vm: |%s|"), - vm_opaque_ref) + vm_ref) admin_context = context.get_admin_context() IPs = db.fixed_ip_get_all_by_instance(admin_context, instance['id']) networks = db.network_get_all_by_instance(admin_context, @@ -731,11 +728,10 @@ class VMOps(object): 'ips': [ip_dict(ip) for ip in network_IPs], 'ip6s': [ip6_dict(ip) for ip in network_IPs]} - self.write_to_param_xenstore(vm_opaque_ref, {location: mapping}) + self.write_to_param_xenstore(vm_ref, {location: mapping}) try: - self.write_to_xenstore(vm_opaque_ref, location, - mapping['location']) + self.write_to_xenstore(vm_ref, location, mapping['location']) except KeyError: # catch KeyError for domid if instance isn't running pass @@ -747,8 +743,8 @@ class VMOps(object): Creates vifs for an instance """ - vm_opaque_ref = self._get_vm_opaque_ref(instance.id) - logging.debug(_("creating vif(s) for vm: |%s|"), vm_opaque_ref) + vm_ref = self._get_vm_opaque_ref(instance.id) + logging.debug(_("creating vif(s) for vm: |%s|"), vm_ref) if networks is None: networks = db.network_get_all_by_instance(admin_context, instance['id']) @@ -768,12 +764,8 @@ class VMOps(object): except AttributeError: device = "0" - VMHelper.create_vif( - self._session, - vm_opaque_ref, - network_ref, - instance.mac_address, - device) + VMHelper.create_vif(self._session, vm_ref, network_ref, + instance.mac_address, device) def reset_network(self, instance): """ @@ -837,9 +829,9 @@ class VMOps(object): Any errors raised by the plugin will in turn raise a RuntimeError here. """ instance_id = vm.id - vm = self._get_vm_opaque_ref(vm) - rec = self._session.get_xenapi().VM.get_record(vm) - args = {'dom_id': rec['domid'], 'path': path} + vm_ref = self._get_vm_opaque_ref(vm) + vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) + args = {'dom_id': vm_rec['domid'], 'path': path} args.update(addl_args) try: task = self._session.async_call_plugin(plugin, method, args) @@ -919,9 +911,9 @@ class VMOps(object): value for 'keys' is passed, the returned dict is filtered to only return the values for those keys. """ - vm = self._get_vm_opaque_ref(instance_or_vm) + vm_ref = self._get_vm_opaque_ref(instance_or_vm) data = self._session.call_xenapi_request('VM.get_xenstore_data', - (vm, )) + (vm_ref, )) ret = {} if keys is None: keys = data.keys() @@ -939,11 +931,11 @@ class VMOps(object): """Takes a key/value pair and adds it to the xenstore parameter record for the given vm instance. If the key exists in xenstore, it is overwritten""" - vm = self._get_vm_opaque_ref(instance_or_vm) + vm_ref = self._get_vm_opaque_ref(instance_or_vm) self.remove_from_param_xenstore(instance_or_vm, key) jsonval = json.dumps(val) self._session.call_xenapi_request('VM.add_to_xenstore_data', - (vm, key, jsonval)) + (vm_ref, key, jsonval)) def write_to_param_xenstore(self, instance_or_vm, mapping): """Takes a dict and writes each key/value pair to the xenstore @@ -958,14 +950,14 @@ class VMOps(object): them from the xenstore parameter record data for the given VM. If the key doesn't exist, the request is ignored. """ - vm = self._get_vm_opaque_ref(instance_or_vm) + vm_ref = self._get_vm_opaque_ref(instance_or_vm) if isinstance(key_or_keys, basestring): keys = [key_or_keys] else: keys = key_or_keys for key in keys: self._session.call_xenapi_request('VM.remove_from_xenstore_data', - (vm, key)) + (vm_ref, key)) def clear_param_xenstore(self, instance_or_vm): """Removes all data from the xenstore parameter record for this VM.""" -- cgit From 616723fe4e7d52b0b8ddafda10fcfe07a87609c8 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 10 Mar 2011 14:53:13 -0500 Subject: add docstring --- nova/compute/api.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'nova') diff --git a/nova/compute/api.py b/nova/compute/api.py index 2766ddc9c..efa051d10 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -81,6 +81,11 @@ class API(base.Base): {"method": "get_network_topic", "args": {'fake': 1}}) def _check_personality_file_quota(self, context, personality_files): + """ + Enforce quota limits on personality files + + Raises a QuotaError if any limit is exceeded + """ limit = quota.allowed_personality_files(context) if len(personality_files) > limit: raise quota.QuotaError(_("Personality limit exceeded. You can " -- cgit From 03e5b8f7c4e1afc6637774acb3d28100035cd323 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Thu, 10 Mar 2011 20:04:21 +0000 Subject: Partial revert of one conversion due to phantom magic exception from SQLAlchemy in unrelated code; convert all deletes --- nova/db/sqlalchemy/api.py | 65 ++++++++++++++++++++++++++++++----------------- 1 file changed, 41 insertions(+), 24 deletions(-) (limited to 'nova') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 2b60ab7d5..31adb33ee 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -701,11 +701,21 @@ def instance_data_get_for_project(context, project_id): def instance_destroy(context, instance_id): session = get_session() with session.begin(): - session.query(models.Instance).\ - filter_by(id=instance_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': models.Instance.updated_at + 0}) + session.execute('update instances set deleted=1,' + 'deleted_at=:at where id=:id', + {'id': instance_id, + 'at': datetime.datetime.utcnow()}) + # NOTE(klmitch): for some reason, using the SQLAlchemy code + # here instead of the direct SQL update above causes the + # test_run_terminate_timestamps test (and only that one) to + # fail with an obscure TypeError exception from deep within + # SQLAlchemy; the nearest nova function in the traceback is + # instance_get() + # session.query(models.Instance).\ + # filter_by(id=instance_id).\ + # update({'deleted': 1, + # 'deleted_at': datetime.datetime.utcnow(), + # 'updated_at': models.Instance.updated_at + 0}) session.query(models.SecurityGroupInstanceAssociation).\ filter_by(instance_id=instance_id).\ update({'deleted': 1, @@ -1837,12 +1847,15 @@ def user_create(_context, values): def user_delete(context, id): session = get_session() with session.begin(): - session.execute('delete from user_project_association ' - 'where user_id=:id', {'id': id}) - session.execute('delete from user_role_association ' - 'where user_id=:id', {'id': id}) - session.execute('delete from user_project_role_association ' - 'where user_id=:id', {'id': id}) + session.query(models.UserProjectAssociation).\ + filter_by(user_id=id).\ + delete() + session.query(models.UserRoleAssociation).\ + filter_by(user_id=id).\ + delete() + session.query(models.UserProjectRoleAssociation).\ + filter_by(user_id=id).\ + delete() user_ref = user_get(context, id, session=session) session.delete(user_ref) @@ -1933,10 +1946,12 @@ def project_update(context, project_id, values): def project_delete(context, id): session = get_session() with session.begin(): - session.execute('delete from user_project_association ' - 'where project_id=:id', {'id': id}) - session.execute('delete from user_project_role_association ' - 'where project_id=:id', {'id': id}) + session.query(models.UserProjectAssociation).\ + filter_by(project_id=id).\ + delete() + session.query(models.UserProjectRoleAssociation).\ + filter_by(project_id=id).\ + delete() project_ref = project_get(context, id, session=session) session.delete(project_ref) @@ -1961,11 +1976,11 @@ def user_get_roles_for_project(context, user_id, project_id): def user_remove_project_role(context, user_id, project_id, role): session = get_session() with session.begin(): - session.execute('delete from user_project_role_association where ' - 'user_id=:user_id and project_id=:project_id and ' - 'role=:role', {'user_id': user_id, - 'project_id': project_id, - 'role': role}) + session.query(models.UserProjectRoleAssociation).\ + filter_by(user_id=user_id).\ + filter_by(project_id=project_id).\ + filter_by(role=role).\ + delete() def user_remove_role(context, user_id, role): @@ -2116,8 +2131,9 @@ def console_delete(context, console_id): session = get_session() with session.begin(): # consoles are meant to be transient. (mdragon) - session.execute('delete from consoles ' - 'where id=:id', {'id': console_id}) + session.query(models.Console).\ + filter_by(id=console_id).\ + delete() def console_get_by_pool_instance(context, pool_id, instance_id): @@ -2273,8 +2289,9 @@ def zone_update(context, zone_id, values): def zone_delete(context, zone_id): session = get_session() with session.begin(): - session.execute('delete from zones ' - 'where id=:id', {'id': zone_id}) + session.query(models.Zone).\ + filter_by(id=zone_id).\ + delete() @require_admin_context -- cgit From c177074649055f1da2ca97eb3c07139571d4a664 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Thu, 10 Mar 2011 12:10:49 -0800 Subject: Implements basic OpenStack API client, ready to support API tests --- nova/tests/integrated/__init__.py | 20 +++ nova/tests/integrated/api/__init__.py | 20 +++ nova/tests/integrated/api/client.py | 224 ++++++++++++++++++++++++++++++++++ 3 files changed, 264 insertions(+) create mode 100644 nova/tests/integrated/__init__.py create mode 100644 nova/tests/integrated/api/__init__.py create mode 100644 nova/tests/integrated/api/client.py (limited to 'nova') diff --git a/nova/tests/integrated/__init__.py b/nova/tests/integrated/__init__.py new file mode 100644 index 000000000..10e0a91d7 --- /dev/null +++ b/nova/tests/integrated/__init__.py @@ -0,0 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +:mod:`integrated` -- Tests whole systems, using mock services where needed +================================= +""" diff --git a/nova/tests/integrated/api/__init__.py b/nova/tests/integrated/api/__init__.py new file mode 100644 index 000000000..5798ab3d1 --- /dev/null +++ b/nova/tests/integrated/api/__init__.py @@ -0,0 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +:mod:`api` -- OpenStack API client, for testing rather than production +================================= +""" diff --git a/nova/tests/integrated/api/client.py b/nova/tests/integrated/api/client.py new file mode 100644 index 000000000..5ab247fab --- /dev/null +++ b/nova/tests/integrated/api/client.py @@ -0,0 +1,224 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import httplib +import urlparse + +from nova import log as logging + + +LOG = logging.getLogger('nova.tests.api') + + +class OpenstackApiException(Exception): + def __init__(self, message=None, response=None): + self.response = response + if not message: + message = 'Unspecified error' + + if response: + _status = response.status + _body = response.read() + + message = _('%(message)s\nStatus Code: %(_status)s\n' + 'Body: %(_body)s') % locals() + + super(OpenstackApiException, self).__init__(message) + + +class OpenstackApiAuthenticationException(OpenstackApiException): + def __init__(self, response=None, message=None): + if not message: + message = _("Authentication error") + super(OpenstackApiAuthenticationException, self).__init__(message, + response) + + +class OpenstackApiNotFoundException(OpenstackApiException): + def __init__(self, response=None, message=None): + if not message: + message = _("Item not found") + super(OpenstackApiNotFoundException, self).__init__(message, response) + + +class TestOpenStackClient(object): + """ A really basic OpenStack API client that is under our control, + so we can make changes / insert hooks for testing""" + + def __init__(self, auth_user, auth_key, auth_uri): + super(TestOpenStackClient, self).__init__() + self.auth_result = None + self.auth_user = auth_user + self.auth_key = auth_key + self.auth_uri = auth_uri + + def request(self, url, method='GET', body=None, headers=None): + if headers is None: + headers = {} + + parsed_url = urlparse.urlparse(url) + port = parsed_url.port + hostname = parsed_url.hostname + scheme = parsed_url.scheme + + if scheme == 'http': + conn = httplib.HTTPConnection(hostname, + port=port) + elif scheme == 'https': + conn = httplib.HTTPSConnection(hostname, + port=port) + else: + raise OpenstackApiException("Unknown scheme: %s" % url) + + relative_url = parsed_url.path + if parsed_url.query: + relative_url = relative_url + parsed_url.query + LOG.info(_("Doing %(method)s on %(relative_url)s") % locals()) + if body: + LOG.info(_("Body: %s") % body) + + conn.request(method, relative_url, body, headers) + response = conn.getresponse() + return response + + def _authenticate(self): + if self.auth_result: + return self.auth_result + + headers = {'X-Auth-User': self.auth_user, + 'X-Auth-Key': self.auth_key} + response = self.request(self.auth_uri, + headers=headers) + if not response.status in [204]: + raise OpenstackApiAuthenticationException(response=response) + + auth_headers = {} + for k, v in response.getheaders(): + auth_headers[k] = v + + self.auth_result = auth_headers + return self.auth_result + + def api_request(self, relative_uri, check_response_status=None, **kwargs): + auth_result = self._authenticate() + + base_uri = auth_result['X-Server-Management-Url'] + full_uri = base_uri + relative_uri + + headers = kwargs.setdefault('headers', {}) + headers['X-Auth-Token'] = auth_result['X-Auth-Token'] + + LOG.debug(_("HTTP request on %s") % (relative_uri)) + + response = self.request(full_uri, **kwargs) + + LOG.debug(_("Response => code %s") % (response.status)) + + if check_response_status: + if not response.status in check_response_status: + if response.status == 404: + raise OpenstackApiNotFoundException(response=response) + else: + raise OpenstackApiException( + message=_("Unexpected status code"), + response=response) + + return response + + def _decode_json(self, response): + body = response.read() + LOG.debug(_("Decoding JSON: %s") % (body)) + return json.loads(body) + + def api_get(self, relative_uri, **kwargs): + kwargs.setdefault('check_response_status', [200]) + response = self.api_request(relative_uri, **kwargs) + return self._decode_json(response) + + def api_post(self, relative_uri, body, **kwargs): + kwargs['method'] = 'POST' + if body: + headers = kwargs.setdefault('headers', {}) + headers['Content-Type'] = 'application/json' + kwargs['body'] = json.dumps(body) + + kwargs.setdefault('check_response_status', [200]) + response = self.api_request(relative_uri, **kwargs) + return self._decode_json(response) + + def api_delete(self, relative_uri, **kwargs): + kwargs['method'] = 'DELETE' + kwargs.setdefault('check_response_status', [200, 202]) + return self.api_request(relative_uri, **kwargs) + + def get_keys_detail(self): + return self.api_get('/keys/detail')['keys'] + + def post_key(self, key): + return self.api_post('/keys', key)['key'] + + def delete_key(self, key_id): + return self.api_delete('/keys/%s' % key_id) + + def get_volume(self, volume_id): + return self.api_get('/volumes/%s' % volume_id)['volume'] + + def get_volumes_detail(self): + return self.api_get('/volumes/detail')['volumes'] + + def post_volume(self, volume): + return self.api_post('/volumes', volume)['volume'] + + def delete_volume(self, volume_id): + return self.api_delete('/volumes/%s' % volume_id) + + def get_server(self, server_id): + return self.api_get('/servers/%s' % server_id)['server'] + + def get_servers(self, detail=True): + rel_url = '/servers/detail' if detail else '/servers' + return self.api_get(rel_url)['servers'] + + def post_server(self, server): + return self.api_post('/servers', server)['server'] + + def delete_server(self, server_id): + return self.api_delete('/servers/%s' % server_id) + + def get_image(self, image_id): + return self.api_get('/images/%s' % image_id)['image'] + + def get_images_detail(self): + return self.api_get('/images/detail')['images'] + + def post_image(self, image): + return self.api_post('/images', image)['image'] + + def delete_image(self, image_id): + return self.api_delete('/images/%s' % image_id) + + def get_flavor(self, flavor_id): + return self.api_get('/flavors/%s' % flavor_id)['flavor'] + + def get_flavors_detail(self): + return self.api_get('/flavors/detail')['flavors'] + + def post_flavor(self, flavor): + return self.api_post('/flavors', flavor)['flavor'] + + def delete_flavor(self, flavor_id): + return self.api_delete('/flavors/%s' % flavor_id) -- cgit From 8aabc32a69bf47075a3fd8e677d1bd70cbbca339 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 21:13:07 +0100 Subject: Add basic test case. --- nova/tests/test_cloud.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) (limited to 'nova') diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index cf8ee7eff..db7c15aeb 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -353,6 +353,18 @@ class CloudTestCase(test.TestCase): self.assertEqual('', img.metadata['description']) shutil.rmtree(pathdir) + def test_metadata_works_without_kernel_and_ramdisk(self): + inst = db.instance_create(self.context, {'host': self.compute.host, + 'vcpus': 2, + 'image_id': '123456', + 'user_data': '' }) + fixed = self.network.allocate_fixed_ip(self.context, inst['id']) + try: + self.cloud.get_metadata(fixed) + finally: + self.network.deallocate_fixed_ip(self.context, fixed) + db.instance_destroy(self.context, inst['id']) + def test_update_of_instance_display_fields(self): inst = db.instance_create(self.context, {}) ec2_id = ec2utils.id_to_ec2_id(inst['id']) -- cgit From 11f2d788fd63c66af0e992f7b75b61273c059bcb Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 21:31:47 +0100 Subject: PEP8 --- nova/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/utils.py b/nova/utils.py index 3008a512e..87e726394 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -166,9 +166,9 @@ def execute(*cmd, **kwargs): stdout=stdout, stderr=stderr, cmd=' '.join(cmd)) - # NOTE(termie): this appears to be necessary to let the subprocess call - # clean something up in between calls, without it two - # execute calls in a row hangs the second one + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one greenthread.sleep(0) return result except ProcessExecutionError: -- cgit From bd06f0ac0d0d3e3c9d7b296c5fe4bb8a0dd44c89 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Thu, 10 Mar 2011 20:36:36 +0000 Subject: Last un-magiced session.execute() replaced with SQLAlchemy code... --- nova/db/sqlalchemy/api.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'nova') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 31adb33ee..88125aaf5 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -579,16 +579,17 @@ def fixed_ip_disassociate_all_by_timeout(_context, host, time): session = get_session() # NOTE(vish): The nested select is because sqlite doesn't support # JOINs in UPDATEs. - result = session.execute('UPDATE fixed_ips SET instance_id = NULL, ' - 'leased = 0 ' - 'WHERE network_id IN (SELECT id FROM networks ' - 'WHERE host = :host) ' - 'AND updated_at < :time ' - 'AND instance_id IS NOT NULL ' - 'AND allocated = 0', - {'host': host, - 'time': time}) - return result.rowcount + inner_q = session.query(models.Network.id).\ + filter_by(host=host).\ + subquery() + result = session.query(models.FixedIp).\ + filter(models.FixedIp.network_id.in_(inner_q)).\ + filter(models.FixedIp.updated_at < time).\ + filter(models.FixedIp.instance_id != None).\ + filter_by(allocated=0).\ + update({'instance_id': None, + 'leased': 0}) + return result @require_admin_context -- cgit From 6b4beef8093b4a7b4d42818567b5afb023af9251 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Thu, 10 Mar 2011 12:55:06 -0800 Subject: Don't wrap keys and volumes till they're in the API --- nova/tests/integrated/api/client.py | 21 --------------------- 1 file changed, 21 deletions(-) (limited to 'nova') diff --git a/nova/tests/integrated/api/client.py b/nova/tests/integrated/api/client.py index 5ab247fab..8ec46b5ae 100644 --- a/nova/tests/integrated/api/client.py +++ b/nova/tests/integrated/api/client.py @@ -165,27 +165,6 @@ class TestOpenStackClient(object): kwargs.setdefault('check_response_status', [200, 202]) return self.api_request(relative_uri, **kwargs) - def get_keys_detail(self): - return self.api_get('/keys/detail')['keys'] - - def post_key(self, key): - return self.api_post('/keys', key)['key'] - - def delete_key(self, key_id): - return self.api_delete('/keys/%s' % key_id) - - def get_volume(self, volume_id): - return self.api_get('/volumes/%s' % volume_id)['volume'] - - def get_volumes_detail(self): - return self.api_get('/volumes/detail')['volumes'] - - def post_volume(self, volume): - return self.api_post('/volumes', volume)['volume'] - - def delete_volume(self, volume_id): - return self.api_delete('/volumes/%s' % volume_id) - def get_server(self, server_id): return self.api_get('/servers/%s' % server_id)['server'] -- cgit From f81d925f86670e3ed32d815c219824f627d82bc2 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Thu, 10 Mar 2011 13:51:26 -0800 Subject: Better logging, be more careful about when we throw login errors re bug732866 --- nova/tests/integrated/api/client.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) (limited to 'nova') diff --git a/nova/tests/integrated/api/client.py b/nova/tests/integrated/api/client.py index 8ec46b5ae..d424a6428 100644 --- a/nova/tests/integrated/api/client.py +++ b/nova/tests/integrated/api/client.py @@ -99,11 +99,18 @@ class TestOpenStackClient(object): if self.auth_result: return self.auth_result + auth_uri = self.auth_uri headers = {'X-Auth-User': self.auth_user, 'X-Auth-Key': self.auth_key} - response = self.request(self.auth_uri, + response = self.request(auth_uri, headers=headers) - if not response.status in [204]: + + http_status = response.status + LOG.debug(_("%(auth_uri)s => code %(http_status)s") % locals()) + + # Until bug732866 is fixed, we can't check this properly... + #if http_status == 401: + if http_status != 204: raise OpenstackApiAuthenticationException(response=response) auth_headers = {} @@ -116,21 +123,21 @@ class TestOpenStackClient(object): def api_request(self, relative_uri, check_response_status=None, **kwargs): auth_result = self._authenticate() - base_uri = auth_result['X-Server-Management-Url'] + #NOTE(justinsb): httplib 'helpfully' converts headers to lower case + base_uri = auth_result['x-server-management-url'] full_uri = base_uri + relative_uri headers = kwargs.setdefault('headers', {}) - headers['X-Auth-Token'] = auth_result['X-Auth-Token'] - - LOG.debug(_("HTTP request on %s") % (relative_uri)) + headers['X-Auth-Token'] = auth_result['x-auth-token'] response = self.request(full_uri, **kwargs) - - LOG.debug(_("Response => code %s") % (response.status)) + + http_status = response.status + LOG.debug(_("%(relative_uri)s => code %(http_status)s") % locals()) if check_response_status: - if not response.status in check_response_status: - if response.status == 404: + if not http_status in check_response_status: + if http_status == 404: raise OpenstackApiNotFoundException(response=response) else: raise OpenstackApiException( @@ -181,8 +188,9 @@ class TestOpenStackClient(object): def get_image(self, image_id): return self.api_get('/images/%s' % image_id)['image'] - def get_images_detail(self): - return self.api_get('/images/detail')['images'] + def get_images(self, detail=True): + rel_url = '/images/detail' if detail else '/images' + return self.api_get(rel_url)['images'] def post_image(self, image): return self.api_post('/images', image)['image'] @@ -193,8 +201,9 @@ class TestOpenStackClient(object): def get_flavor(self, flavor_id): return self.api_get('/flavors/%s' % flavor_id)['flavor'] - def get_flavors_detail(self): - return self.api_get('/flavors/detail')['flavors'] + def get_flavors(self, detail=True): + rel_url = '/flavors/detail' if detail else '/flavors' + return self.api_get(rel_url)['flavors'] def post_flavor(self, flavor): return self.api_post('/flavors', flavor)['flavor'] -- cgit From f110e718807ea4747a0ff95138c488961257aa7f Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Thu, 10 Mar 2011 13:56:24 -0800 Subject: pep8 fun --- nova/tests/integrated/api/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'nova') diff --git a/nova/tests/integrated/api/client.py b/nova/tests/integrated/api/client.py index d424a6428..da8d87e07 100644 --- a/nova/tests/integrated/api/client.py +++ b/nova/tests/integrated/api/client.py @@ -104,7 +104,7 @@ class TestOpenStackClient(object): 'X-Auth-Key': self.auth_key} response = self.request(auth_uri, headers=headers) - + http_status = response.status LOG.debug(_("%(auth_uri)s => code %(http_status)s") % locals()) @@ -131,7 +131,7 @@ class TestOpenStackClient(object): headers['X-Auth-Token'] = auth_result['x-auth-token'] response = self.request(full_uri, **kwargs) - + http_status = response.status LOG.debug(_("%(relative_uri)s => code %(http_status)s") % locals()) -- cgit From be66b329d5b94ffbfb782355ef342eadbaed72a5 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Thu, 10 Mar 2011 22:14:53 +0000 Subject: Fix a fer nits jaypipes found in review. --- nova/api/openstack/accounts.py | 18 +++++++++++++++--- nova/api/openstack/users.py | 4 ++-- nova/tests/api/openstack/test_accounts.py | 6 +++--- nova/tests/api/openstack/test_auth.py | 15 ++++----------- nova/tests/api/openstack/test_users.py | 2 +- 5 files changed, 25 insertions(+), 20 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/accounts.py b/nova/api/openstack/accounts.py index 3b90d2776..dd88c3390 100644 --- a/nova/api/openstack/accounts.py +++ b/nova/api/openstack/accounts.py @@ -21,6 +21,7 @@ from nova import log as logging from nova import wsgi from nova.auth import manager +from nova.api.openstack import faults FLAGS = flags.FLAGS LOG = logging.getLogger('nova.api.openstack') @@ -44,11 +45,17 @@ class Controller(wsgi.Controller): self.manager = manager.AuthManager() def _check_admin(self, context): - """ We cannot depend on the db layer to check for admin access - for the auth manager, so we do it here """ + """We cannot depend on the db layer to check for admin access + for the auth manager, so we do it here""" if not context.is_admin: raise exception.NotAuthorized(_("Not admin user.")) + def index(self, req): + raise faults.Fault(exc.HTTPNotImplemented()) + + def detail(self, req): + raise faults.Fault(exc.HTTPNotImplemented()) + def show(self, req, id): """Return data about the given account id""" account = self.manager.get_project(id) @@ -59,8 +66,13 @@ class Controller(wsgi.Controller): self.manager.delete_project(id) return {} + def create(self, req): + """We use update with create-or-update semantics + because the id comes from an external source""" + raise faults.Fault(exc.HTTPNotImplemented()) + def update(self, req, id): - """ This is really create or update. """ + """This is really create or update.""" self._check_admin(req.environ['nova.context']) env = self._deserialize(req.body, req) description = env['account'].get('description') diff --git a/nova/api/openstack/users.py b/nova/api/openstack/users.py index 83ebec964..5bb20a718 100644 --- a/nova/api/openstack/users.py +++ b/nova/api/openstack/users.py @@ -45,8 +45,8 @@ class Controller(wsgi.Controller): self.manager = manager.AuthManager() def _check_admin(self, context): - """ We cannot depend on the db layer to check for admin access - for the auth manager, so we do it here """ + """We cannot depend on the db layer to check for admin access + for the auth manager, so we do it here""" if not context.is_admin: raise exception.NotAuthorized(_("Not admin user")) diff --git a/nova/tests/api/openstack/test_accounts.py b/nova/tests/api/openstack/test_accounts.py index 746f02f57..78fceb47c 100644 --- a/nova/tests/api/openstack/test_accounts.py +++ b/nova/tests/api/openstack/test_accounts.py @@ -14,9 +14,10 @@ # under the License. +import json + import stubout import webob -import json import nova.api import nova.api.openstack.auth @@ -47,8 +48,7 @@ class AccountsTest(test.TestCase): fake_init) self.stubs.Set(nova.api.openstack.accounts.Controller, '_check_admin', fake_admin_check) - fakes.FakeAuthManager.auth_data = {} - fakes.FakeAuthManager.projects = {} + fakes.FakeAuthManager.clear_fakes() fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) fakes.stub_out_rate_limiting(self.stubs) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 49f90879d..437a79ec5 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -51,9 +51,7 @@ class Test(test.TestCase): def test_authorize_user(self): f = fakes.FakeAuthManager() - u = nova.auth.manager.User(1, 'herp', None, None, None) - f.add_user('derp', u) - f.create_project('test', u) + f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' @@ -67,9 +65,7 @@ class Test(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - u = nova.auth.manager.User(1, 'herp', None, None, None) - f.add_user('derp', u) - f.create_project('test', u) + f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) req.headers['X-Auth-User'] = 'herp' @@ -86,7 +82,7 @@ class Test(test.TestCase): token = result.headers['X-Auth-Token'] self.stubs.Set(nova.api.openstack, 'APIRouter', fakes.FakeRouter) - req = webob.Request.blank('/v1.0/test/fake') + req = webob.Request.blank('/v1.0/fake') req.headers['X-Auth-Token'] = token result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '200 OK') @@ -180,9 +176,6 @@ class TestLimiter(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - u = nova.auth.manager.User(1, 'herp', None, None, None) - f.add_user('derp', u) - f.create_project('test', u) f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) req = webob.Request.blank('/v1.0/') @@ -194,7 +187,7 @@ class TestLimiter(test.TestCase): token = result.headers['X-Auth-Token'] self.stubs.Set(nova.api.openstack, 'APIRouter', fakes.FakeRouter) - req = webob.Request.blank('/v1.0/test/fake') + req = webob.Request.blank'/v1.0/fake') req.method = 'POST' req.headers['X-Auth-Token'] = token result = req.get_response(fakes.wsgi_app()) diff --git a/nova/tests/api/openstack/test_users.py b/nova/tests/api/openstack/test_users.py index 14c7897f0..1edefe713 100644 --- a/nova/tests/api/openstack/test_users.py +++ b/nova/tests/api/openstack/test_users.py @@ -13,10 +13,10 @@ # License for the specific language governing permissions and limitations # under the License. +import json import stubout import webob -import json import nova.api import nova.api.openstack.auth -- cgit From f251ef70bf83eebce0f851f8a1b052174be1d615 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Thu, 10 Mar 2011 22:20:51 +0000 Subject: fix minor typo --- nova/tests/api/openstack/test_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 437a79ec5..ff8d42a14 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -187,7 +187,7 @@ class TestLimiter(test.TestCase): token = result.headers['X-Auth-Token'] self.stubs.Set(nova.api.openstack, 'APIRouter', fakes.FakeRouter) - req = webob.Request.blank'/v1.0/fake') + req = webob.Request.blank('/v1.0/fake') req.method = 'POST' req.headers['X-Auth-Token'] = token result = req.get_response(fakes.wsgi_app()) -- cgit From 7e95a65ccec2336176f389d614a85c9e70da374d Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Thu, 10 Mar 2011 22:33:45 +0000 Subject: re-added a test change I removed thinking it was related to removed code. It wasn't :> --- nova/tests/api/openstack/test_auth.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index ff8d42a14..aaaa4e415 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -65,7 +65,9 @@ class Test(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) + u = nova.auth.manager.User(1, 'herp', None, None, None) + f.add_user('derp', u) + f.create_project('test', u) req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) req.headers['X-Auth-User'] = 'herp' @@ -176,7 +178,9 @@ class TestLimiter(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) + u = nova.auth.manager.User(1, 'herp', None, None, None) + f.add_user('derp', u) + f.create_project('test', u) req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' -- cgit From c967679fa8144af57d79d89666ee29a0241d38a9 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 10 Mar 2011 17:36:41 -0500 Subject: switch to a more consistent usage of onset_files variable names --- nova/api/openstack/servers.py | 59 ++++++++++++++++++++----------- nova/compute/api.py | 38 ++++++++------------ nova/compute/manager.py | 2 +- nova/quota.py | 30 ++++++++-------- nova/tests/api/openstack/test_servers.py | 48 ++++++++++++------------- nova/tests/test_quota.py | 60 ++++++++++++++++---------------- 6 files changed, 124 insertions(+), 113 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 7bef8eb32..adb5c5f99 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -169,20 +169,23 @@ class Controller(wsgi.Controller): metadata.append({'key': k, 'value': v}) personality = env['server'].get('personality', []) - personality_files = self._get_personality_files(personality) - - instances = self.compute_api.create( - context, - instance_types.get_by_flavor_id(env['server']['flavorId']), - image_id, - kernel_id=kernel_id, - ramdisk_id=ramdisk_id, - display_name=env['server']['name'], - display_description=env['server']['name'], - key_name=key_pair['name'], - key_data=key_pair['public_key'], - metadata=metadata, - personality_files=personality_files) + onset_files = self._get_onset_files(personality) + + try: + instances = self.compute_api.create( + context, + instance_types.get_by_flavor_id(env['server']['flavorId']), + image_id, + kernel_id=kernel_id, + ramdisk_id=ramdisk_id, + display_name=env['server']['name'], + display_description=env['server']['name'], + key_name=key_pair['name'], + key_data=key_pair['public_key'], + metadata=metadata, + onset_files=onset_files) + except QuotaError as error: + self._handle_quota_error(error) server = _translate_keys(instances[0]) password = "%s%s" % (server['server']['name'][:4], @@ -204,15 +207,15 @@ class Controller(wsgi.Controller): else: return self._deserialize(request.body, request.get_content_type()) - def _get_personality_files(self, personality): + def _get_onset_files(self, personality): """ - Create a list of personality files from the personality attribute + Create a list of onset files from the personality attribute - At this time, personality_files must be formatted as a list of + At this time, onset_files must be formatted as a list of (file_path, file_content) pairs for compatibility with the underlying compute service. """ - personality_files = [] + onset_files = [] for item in personality: try: path = item['path'] @@ -227,8 +230,24 @@ class Controller(wsgi.Controller): except TypeError: msg = 'Personality content for %s cannot be decoded' % path raise exc.HTTPBadRequest(explanation=msg) - personality_files.append((path, contents)) - return personality_files + onset_files.append((path, contents)) + return onset_files + + def _handle_quota_errors(self, error): + """ + Reraise quota errors as api-specific http exceptions + """ + if error.code == "OnsetFileLimitExceeded": + expl = "Personality file limit exceeded" + raise exc.HTTPBadRequest(explanation=expl) + if error.code == "OnsetFilePathLimitExceeded": + expl = "Personality file path too long" + raise exc.HTTPBadRequest(explanation=expl) + if error.code == "OnsetFileContentLimitExceeded": + expl = "Personality file content too long" + raise exc.HTTPBadRequest(explanation=expl) + # if the original error is okay, just reraise it + raise error def update(self, req, id): """ Updates the server name or password """ diff --git a/nova/compute/api.py b/nova/compute/api.py index b97cadf61..140bbb3aa 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -80,30 +80,23 @@ class API(base.Base): topic, {"method": "get_network_topic", "args": {'fake': 1}}) - def _check_personality_file_quota(self, context, personality_files): + def _check_onset_file_quota(self, context, onset_files): """ - Enforce quota limits on personality files + Enforce quota limits on onset files Raises a QuotaError if any limit is exceeded """ - limit = quota.allowed_personality_files(context) - if len(personality_files) > limit: - raise quota.QuotaError(_("Personality limit exceeded. You can " - "only have %d personalities when " - "creating an instance.") % limit, - "PersonalityLimitExceeded") - path_limit = quota.allowed_personality_path_bytes(context) - content_limit = quota.allowed_personality_content_bytes(context) - for path, content in personality_files: + limit = quota.allowed_onset_files(context) + if len(onset_files) > limit: + raise quota.QuotaError(code="OnsetFileLimitExceeded") + path_limit = quota.allowed_onset_file_path_bytes(context) + content_limit = quota.allowed_onset_file_content_bytes(context) + for path, content in onset_files: if len(path) > path_limit: - raise quota.QuotaError( - _("Personality file path limit exceeded."), - "PersonalityLimitExceeded") + raise quota.QuotaError(code="OnsetFilePathLimitExceeded") if len(content) > content_limit: - raise quota.QuotaError( - _("Personality file content limit exceeded."), - "PersonalityLimitExceeded") - return personality_files + raise quota.QuotaError(code="OnsetFileContentLimitExceeded") + return onset_files def create(self, context, instance_type, image_id, kernel_id=None, ramdisk_id=None, @@ -111,7 +104,7 @@ class API(base.Base): display_name='', display_description='', key_name=None, key_data=None, security_group='default', availability_zone=None, user_data=None, metadata=[], - personality_files=None): + onset_files=None): """Create the number of instances requested if quota and other arguments check out ok.""" @@ -149,9 +142,8 @@ class API(base.Base): LOG.warn(msg) raise quota.QuotaError(msg, "MetadataLimitExceeded") - if personality_files is not None: - personality_files = \ - self._check_personality_file_quota(context, personality_files) + if onset_files is not None: + onset_files = self._check_onset_file_quota(context, onset_files) image = self.image_service.show(context, image_id) if kernel_id is None: @@ -248,7 +240,7 @@ class API(base.Base): "args": {"topic": FLAGS.compute_topic, "instance_id": instance_id, "availability_zone": availability_zone, - "personality_files": personality_files}}) + "onset_files": onset_files}}) for group_id in security_groups: self.trigger_security_group_members_refresh(elevated, group_id) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index d87290aae..601bb3084 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -173,7 +173,7 @@ class ComputeManager(manager.Manager): """Launch a new instance with specified options.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) - instance_ref.onset_files = kwargs.get('personality_files', []) + instance_ref.onset_files = kwargs.get('onset_files', []) if instance_ref['name'] in self.driver.list_instances(): raise exception.Error(_("Instance has already been created")) LOG.audit(_("instance %s: starting..."), instance_id, diff --git a/nova/quota.py b/nova/quota.py index fdef42bc5..e0fb97542 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -37,12 +37,12 @@ flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') flags.DEFINE_integer('quota_metadata_items', 128, 'number of metadata items allowed per instance') -flags.DEFINE_integer('quota_personality_max_files', 5, - 'number of personality files allowed') -flags.DEFINE_integer('quota_personality_max_content_bytes', 10 * 1024, - 'number of bytes allowed per personality file') -flags.DEFINE_integer('quota_personality_max_path_bytes', 255, - 'number of bytes allowed per personality file path') +flags.DEFINE_integer('quota_max_onset_files', 5, + 'number of onset files allowed') +flags.DEFINE_integer('quota_max_onset_file_content_bytes', 10 * 1024, + 'number of bytes allowed per onset file') +flags.DEFINE_integer('quota_max_onset_file_path_bytes', 255, + 'number of bytes allowed per onset file path') def get_quota(context, project_id): @@ -113,19 +113,19 @@ def allowed_metadata_items(context, num_metadata_items): return min(num_metadata_items, num_allowed_metadata_items) -def allowed_personality_files(context): - """Return the number of personality files allowed""" - return int(FLAGS.quota_personality_max_files) +def allowed_onset_files(context): + """Return the number of onset files allowed""" + return int(FLAGS.quota_max_onset_files) -def allowed_personality_content_bytes(context): - """Return the number of bytes allowed per personality content""" - return int(FLAGS.quota_personality_max_content_bytes) +def allowed_onset_file_content_bytes(context): + """Return the number of bytes allowed per onset file content""" + return int(FLAGS.quota_max_onset_file_content_bytes) -def allowed_personality_path_bytes(context): - """Return the number of bytes allowed in a personality file path""" - return int(FLAGS.quota_personality_max_path_bytes) +def allowed_onset_file_path_bytes(context): + """Return the number of bytes allowed in an onset file path""" + return int(FLAGS.quota_max_onset_file_path_bytes) class QuotaError(exception.ApiError): diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 0561ad499..b6d88d833 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -833,13 +833,13 @@ class TestServerInstanceCreation(test.TestCase): class MockComputeAPI(object): def __init__(self): - self.personality_files = None + self.onset_files = None def create(self, *args, **kwargs): - if 'personality_files' in kwargs: - self.personality_files = kwargs['personality_files'] + if 'onset_files' in kwargs: + self.onset_files = kwargs['onset_files'] else: - self.personality_files = None + self.onset_files = None return [{'id': '1234', 'display_name': 'fakeinstance'}] def set_admin_password(self, *args, **kwargs): @@ -919,46 +919,46 @@ class TestServerInstanceCreation(test.TestCase): request = self._get_create_request_json(body_dict) compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) - return request, response, compute_api.personality_files + return request, response, compute_api.onset_files def _create_instance_with_personality_xml(self, personality): body_dict = self._create_personality_request_dict(personality) request = self._get_create_request_xml(body_dict) compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) - return request, response, compute_api.personality_files + return request, response, compute_api.onset_files def test_create_instance_with_no_personality(self): - request, response, personality_files = \ + request, response, onset_files = \ self._create_instance_with_personality_json(personality=None) self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, []) + self.assertEquals(onset_files, []) def test_create_instance_with_no_personality_xml(self): - request, response, personality_files = \ + request, response, onset_files = \ self._create_instance_with_personality_xml(personality=None) self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, []) + self.assertEquals(onset_files, []) def test_create_instance_with_personality(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Hello, World!"\n' b64contents = base64.b64encode(contents) personality = [(path, b64contents)] - request, response, personality_files = \ + request, response, onset_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, [(path, contents)]) + self.assertEquals(onset_files, [(path, contents)]) def test_create_instance_with_personality_xml(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Hello, World!"\n' b64contents = base64.b64encode(contents) personality = [(path, b64contents)] - request, response, personality_files = \ + request, response, onset_files = \ self._create_instance_with_personality_xml(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, [(path, contents)]) + self.assertEquals(onset_files, [(path, contents)]) def test_create_instance_with_personality_no_path(self): personality = [('/remove/this/path', @@ -969,7 +969,7 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) + self.assertEquals(compute_api.onset_files, None) def _test_create_instance_with_personality_no_path_xml(self): personality = [('/remove/this/path', @@ -980,7 +980,7 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) + self.assertEquals(compute_api.onset_files, None) def test_create_instance_with_personality_no_contents(self): personality = [('/test/path', @@ -991,7 +991,7 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) + self.assertEquals(compute_api.onset_files, None) def test_create_instance_with_personality_not_a_list(self): personality = [('/test/path', base64.b64encode('test\ncontents\n'))] @@ -1002,16 +1002,16 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) + self.assertEquals(compute_api.onset_files, None) def test_create_instance_with_personality_with_non_b64_content(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Oh no!"\n' personality = [(path, contents)] - request, response, personality_files = \ + request, response, onset_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 400) - self.assertEquals(personality_files, None) + self.assertEquals(onset_files, None) def test_create_instance_with_three_personalities(self): files = [ @@ -1022,19 +1022,19 @@ class TestServerInstanceCreation(test.TestCase): personality = [] for path, content in files: personality.append((path, base64.b64encode(content))) - request, response, personality_files = \ + request, response, onset_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, files) + self.assertEquals(onset_files, files) def test_create_instance_personality_empty_content(self): path = '/my/file/path' contents = '' personality = [(path, contents)] - request, response, personality_files = \ + request, response, onset_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, [(path, contents)]) + self.assertEquals(onset_files, [(path, contents)]) def test_create_instance_admin_pass_json(self): request, response, dummy = \ diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 86b1d40fb..d94381aa2 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -200,66 +200,66 @@ class QuotaTestCase(test.TestCase): image_id='fake', metadata=metadata) - def test_allowed_personality_files(self): + def test_allowed_onset_files(self): self.assertEqual( - quota.allowed_personality_files(self.context), - FLAGS.quota_personality_max_files) + quota.allowed_onset_files(self.context), + FLAGS.quota_max_onset_files) - def _create_with_personality(self, files): + def _create_with_onset_files(self, files): api = compute.API(image_service=self.StubImageService()) api.create(self.context, min_count=1, max_count=1, instance_type='m1.small', image_id='fake', - personality_files=files) + onset_files=files) - def test_no_personality_files(self): + def test_no_onset_files(self): api = compute.API(image_service=self.StubImageService()) api.create(self.context, instance_type='m1.small', image_id='fake') - def test_max_personality_files(self): + def test_max_onset_files(self): files = [] - for i in xrange(FLAGS.quota_personality_max_files): + for i in xrange(FLAGS.quota_max_onset_files): files.append(('/my/path%d' % i, 'config = test\n')) - self._create_with_personality(files) # no QuotaError + self._create_with_onset_files(files) # no QuotaError - def test_too_many_personality_files(self): + def test_too_many_onset_files(self): files = [] - for i in xrange(FLAGS.quota_personality_max_files + 1): + for i in xrange(FLAGS.quota_max_onset_files + 1): files.append(('/my/path%d' % i, 'my\ncontent%d\n' % i)) self.assertRaises(quota.QuotaError, - self._create_with_personality, files) + self._create_with_onset_files, files) - def test_allowed_personality_content_bytes(self): + def test_allowed_onset_file_content_bytes(self): self.assertEqual( - quota.allowed_personality_content_bytes(self.context), - FLAGS.quota_personality_max_content_bytes) + quota.allowed_onset_file_content_bytes(self.context), + FLAGS.quota_max_onset_file_content_bytes) - def test_max_personality_content_bytes(self): - max = FLAGS.quota_personality_max_content_bytes + def test_max_onset_file_content_bytes(self): + max = FLAGS.quota_max_onset_file_content_bytes content = ''.join(['a' for i in xrange(max)]) files = [('/test/path', content)] - self._create_with_personality(files) # no QuotaError + self._create_with_onset_files(files) # no QuotaError - def test_too_many_personality_content_bytes(self): - max = FLAGS.quota_personality_max_content_bytes + def test_too_many_onset_file_content_bytes(self): + max = FLAGS.quota_max_onset_file_content_bytes content = ''.join(['a' for i in xrange(max + 1)]) files = [('/test/path', content)] self.assertRaises(quota.QuotaError, - self._create_with_personality, files) + self._create_with_onset_files, files) - def test_allowed_personality_path_bytes(self): + def test_allowed_onset_file_path_bytes(self): self.assertEqual( - quota.allowed_personality_path_bytes(self.context), - FLAGS.quota_personality_max_path_bytes) + quota.allowed_onset_file_path_bytes(self.context), + FLAGS.quota_max_onset_file_path_bytes) - def test_max_personality_path_bytes(self): - max = FLAGS.quota_personality_max_path_bytes + def test_max_onset_file_path_bytes(self): + max = FLAGS.quota_max_onset_file_path_bytes path = ''.join(['a' for i in xrange(max)]) files = [(path, 'config = quotatest')] - self._create_with_personality(files) # no QuotaError + self._create_with_onset_files(files) # no QuotaError - def test_too_many_personality_path_bytes(self): - max = FLAGS.quota_personality_max_path_bytes + def test_too_many_onset_file_path_bytes(self): + max = FLAGS.quota_max_onset_file_path_bytes path = ''.join(['a' for i in xrange(max + 1)]) files = [(path, 'config = quotatest')] self.assertRaises(quota.QuotaError, - self._create_with_personality, files) + self._create_with_onset_files, files) -- cgit From b9a479ffc8e9db0c1888047d7f3df99b3b57b2ec Mon Sep 17 00:00:00 2001 From: Dan Prince Date: Thu, 10 Mar 2011 21:44:01 -0500 Subject: Make linux_net ensure_bridge commands that add and remove ip addr's from devices/bridges work with with the latest utils.execute method (execvp). --- nova/network/linux_net.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 9f9d282b6..e69ed2f75 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -513,11 +513,9 @@ def ensure_bridge(bridge, interface, net_attrs=None): for line in out.split("\n"): fields = line.split() if fields and fields[0] == "inet": - params = ' '.join(fields[1:-1]) - _execute('sudo', 'ip', 'addr', - 'del', params, 'dev', fields[-1]) - _execute('sudo', 'ip', 'addr', - 'add', params, 'dev', bridge) + params = fields[1:-1] + _execute(*_ip_bridge_cmd('del', params, fields[-1])) + _execute(*_ip_bridge_cmd('add', params, bridge)) if gateway: _execute('sudo', 'route', 'add', '0.0.0.0', 'gw', gateway) out, err = _execute('sudo', 'brctl', 'addif', bridge, interface, @@ -739,3 +737,12 @@ def _ra_pid_for(bridge): if os.path.exists(pid_file): with open(pid_file, 'r') as f: return int(f.read()) + + +def _ip_bridge_cmd(action, params, device): + """Build commands to add/del ips to bridges/devices""" + + cmd = ['sudo', 'ip', 'addr', action] + cmd.extend(params) + cmd.extend(['dev', device]) + return cmd -- cgit From 67a8d635af0a64ad220b163c00b96eadf7daf93f Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 11 Mar 2011 09:54:08 +0100 Subject: Make Authors check account for tests being run with different os.getcwd() depending on how they're run. Add missing people to Authors. --- nova/tests/test_misc.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'nova') diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index a658e4978..1fbaf304f 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -24,18 +24,19 @@ from nova.utils import parse_mailmap, str_dict_replace, synchronized class ProjectTestCase(test.TestCase): def test_authors_up_to_date(self): - if os.path.exists('.bzr'): + topdir = os.path.normpath(os.path.dirname(__file__) + '/../../') + if os.path.exists(os.path.join(topdir, '.bzr')): contributors = set() - mailmap = parse_mailmap('.mailmap') + mailmap = parse_mailmap(os.path.join(topdir, '.mailmap')) import bzrlib.workingtree - tree = bzrlib.workingtree.WorkingTree.open('.') + tree = bzrlib.workingtree.WorkingTree.open(topdir) tree.lock_read() try: parents = tree.get_parent_ids() g = tree.branch.repository.get_graph() - for p in parents[1:]: + for p in parents: rev_ids = [r for r, _ in g.iter_ancestry(parents) if r != "null:"] revs = tree.branch.repository.get_revisions(rev_ids) @@ -44,7 +45,8 @@ class ProjectTestCase(test.TestCase): email = author.split(' ')[-1] contributors.add(str_dict_replace(email, mailmap)) - authors_file = open('Authors', 'r').read() + authors_file = open(os.path.join(topdir, 'Authors'), + 'r').read() missing = set() for contributor in contributors: -- cgit From 46d1f6a8c888c1f6fdf12cf26df67eada1e8505b Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 11 Mar 2011 11:24:22 +0100 Subject: Use self.instances.pop in unfilter_instance to make the check/removal atomic. Move the semaphore grab outside the for loop in refresh_security_group_rules to avoid reading a value from self.instances, blocking waiting for the semaphore, having the instance be removed in the mean time, and then add its rules back. --- nova/virt/libvirt_conn.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index b74ed25f9..d82b33ddd 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1238,13 +1238,12 @@ class IptablesFirewallDriver(FirewallDriver): pass def unfilter_instance(self, instance): - if instance['id'] in self.instances: - del self.instances[instance['id']] + if self.instances.pop(instance['id'], False): self.remove_filters_for_instance(instance) self.iptables.apply() else: LOG.info(_('Attempted to unfilter instance %s which is not ' - 'filtered'), instance['id']) + 'filtered'), instance['id']) def prepare_instance_filter(self, instance): self.instances[instance['id']] = instance @@ -1387,11 +1386,11 @@ class IptablesFirewallDriver(FirewallDriver): pass def refresh_security_group_rules(self, security_group): - for instance in self.instances.values(): - # We use the semaphore to make sure noone applies the rule set - # after we've yanked the existing rules but before we've put in - # the new ones. - with self.iptables.semaphore: + # We use the semaphore to make sure noone applies the rule set + # after we've yanked the existing rules but before we've put in + # the new ones. + with self.iptables.semaphore: + for instance in self.instances.values(): self.remove_filters_for_instance(instance) self.add_filters_for_instance(instance) self.iptables.apply() -- cgit From d250d522b5d6c164435fc254223ff9a0f055cf05 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 11 Mar 2011 17:55:28 +0100 Subject: Remove broken test. At least this way, it'll actually fix the problem and be mergable. --- nova/tests/test_cloud.py | 12 ------------ 1 file changed, 12 deletions(-) (limited to 'nova') diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index db7c15aeb..cf8ee7eff 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -353,18 +353,6 @@ class CloudTestCase(test.TestCase): self.assertEqual('', img.metadata['description']) shutil.rmtree(pathdir) - def test_metadata_works_without_kernel_and_ramdisk(self): - inst = db.instance_create(self.context, {'host': self.compute.host, - 'vcpus': 2, - 'image_id': '123456', - 'user_data': '' }) - fixed = self.network.allocate_fixed_ip(self.context, inst['id']) - try: - self.cloud.get_metadata(fixed) - finally: - self.network.deallocate_fixed_ip(self.context, fixed) - db.instance_destroy(self.context, inst['id']) - def test_update_of_instance_display_fields(self): inst = db.instance_create(self.context, {}) ec2_id = ec2utils.id_to_ec2_id(inst['id']) -- cgit From 36b5f7d9cf377ce2a4dcdad07e7e14062cd3ec4d Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 11:22:23 -0600 Subject: Further vmops cleanup --- nova/virt/xenapi/vmops.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'nova') diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 5375df5b4..f012fa446 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -536,10 +536,10 @@ class VMOps(object): """ instance_id = instance.id LOG.info(_("Destroying VM for Instance %(instance_id)s") % locals()) - vm = VMHelper.lookup(self._session, instance.name) - return self._destroy(instance, vm, shutdown=True) + vm_ref = VMHelper.lookup(self._session, instance.name) + return self._destroy(instance, vm_ref, shutdown=True) - def _destroy(self, instance, vm, shutdown=True, + def _destroy(self, instance, vm_ref, shutdown=True, destroy_kernel_ramdisk=True): """ Destroys VM instance by performing: @@ -549,17 +549,17 @@ class VMOps(object): 3. Destroying kernel and ramdisk files (if necessary) 4. Destroying that actual VM record """ - if vm is None: + if vm_ref is None: LOG.warning(_("VM is not present, skipping destroy...")) return if shutdown: - self._shutdown(instance, vm) + self._shutdown(instance, vm_ref) - self._destroy_vdis(instance, vm) + self._destroy_vdis(instance, vm_ref) if destroy_kernel_ramdisk: - self._destroy_kernel_ramdisk(instance, vm) - self._destroy_vm(instance, vm) + self._destroy_kernel_ramdisk(instance, vm_ref) + self._destroy_vm(instance, vm_ref) def _wait_with_callback(self, instance_id, task, callback): ret = None -- cgit From cfc7d21b959bc929295868aeb3e84ea56afbfd9c Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Fri, 11 Mar 2011 17:41:22 +0000 Subject: Discovered literal_column(), which does exactly what I need --- nova/db/sqlalchemy/api.py | 49 +++++++++++++++-------------------------------- 1 file changed, 15 insertions(+), 34 deletions(-) (limited to 'nova') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 88125aaf5..431cf6e8e 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -34,6 +34,7 @@ from sqlalchemy.orm import joinedload from sqlalchemy.orm import joinedload_all from sqlalchemy.sql import exists from sqlalchemy.sql import func +from sqlalchemy.sql.expression import literal_column FLAGS = flags.FLAGS @@ -702,28 +703,16 @@ def instance_data_get_for_project(context, project_id): def instance_destroy(context, instance_id): session = get_session() with session.begin(): - session.execute('update instances set deleted=1,' - 'deleted_at=:at where id=:id', - {'id': instance_id, - 'at': datetime.datetime.utcnow()}) - # NOTE(klmitch): for some reason, using the SQLAlchemy code - # here instead of the direct SQL update above causes the - # test_run_terminate_timestamps test (and only that one) to - # fail with an obscure TypeError exception from deep within - # SQLAlchemy; the nearest nova function in the traceback is - # instance_get() - # session.query(models.Instance).\ - # filter_by(id=instance_id).\ - # update({'deleted': 1, - # 'deleted_at': datetime.datetime.utcnow(), - # 'updated_at': models.Instance.updated_at + 0}) + session.query(models.Instance).\ + filter_by(id=instance_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupInstanceAssociation).\ filter_by(instance_id=instance_id).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': - (models.SecurityGroupInstanceAssociation. - updated_at + 0)}) + 'updated_at': literal_column('updated_at')}) @require_context @@ -969,7 +958,7 @@ def key_pair_destroy_all_by_user(context, user_id): filter_by(user_id=user_id).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': models.KeyPair.updated_at + 0}) + 'updated_at': literal_column('updated_at')}) @require_context @@ -1082,7 +1071,7 @@ def network_disassociate_all(context): session = get_session() session.query(models.Network).\ update({'project_id': None, - 'updated_at': models.Network.updated_at + 0}) + 'updated_at': literal_column('updated_at')}) @require_context @@ -1456,7 +1445,7 @@ def volume_destroy(context, volume_id): filter_by(id=volume_id).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': models.Volume.updated_at + 0}) + 'updated_at': literal_column('updated_at')}) session.query(models.ExportDevice).\ filter_by(volume_id=volume_id).\ update({'volume_id': None}) @@ -1686,22 +1675,17 @@ def security_group_destroy(context, security_group_id): filter_by(id=security_group_id).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': - models.SecurityGroup.updated_at + 0}) + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupInstanceAssociation).\ filter_by(security_group_id=security_group_id).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': - (models.SecurityGroupInstanceAssociation. - updated_at + 0)}) + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupIngressRule).\ filter_by(group_id=security_group_id).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': - (models.SecurityGroupIngressRule. - updated_at + 0)}) + 'updated_at': literal_column('updated_at')}) @require_context @@ -1712,14 +1696,11 @@ def security_group_destroy_all(context, session=None): session.query(models.SecurityGroup).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': - models.SecurityGroup.updated_at + 0}) + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupIngressRule).\ update({'deleted': 1, 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': - (models.SecurityGroupIngressRule. - updated_at + 0)}) + 'updated_at': literal_column('updated_at')}) ################### -- cgit From 195926d0635c0217edccf1cd763425163d3e92e7 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Fri, 11 Mar 2011 19:22:31 +0000 Subject: Minor stylistic updates affecting indentation --- nova/db/sqlalchemy/api.py | 132 +++++++++++++++++++++++----------------------- 1 file changed, 66 insertions(+), 66 deletions(-) (limited to 'nova') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 89745aa95..08bc8fe2f 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -581,15 +581,15 @@ def fixed_ip_disassociate_all_by_timeout(_context, host, time): # NOTE(vish): The nested select is because sqlite doesn't support # JOINs in UPDATEs. inner_q = session.query(models.Network.id).\ - filter_by(host=host).\ - subquery() + filter_by(host=host).\ + subquery() result = session.query(models.FixedIp).\ - filter(models.FixedIp.network_id.in_(inner_q)).\ - filter(models.FixedIp.updated_at < time).\ - filter(models.FixedIp.instance_id != None).\ - filter_by(allocated=0).\ - update({'instance_id': None, - 'leased': 0}) + filter(models.FixedIp.network_id.in_(inner_q)).\ + filter(models.FixedIp.updated_at < time).\ + filter(models.FixedIp.instance_id != None).\ + filter_by(allocated=0).\ + update({'instance_id': None, + 'leased': 0}) return result @@ -704,15 +704,15 @@ def instance_destroy(context, instance_id): session = get_session() with session.begin(): session.query(models.Instance).\ - filter_by(id=instance_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + filter_by(id=instance_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupInstanceAssociation).\ - filter_by(instance_id=instance_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + filter_by(instance_id=instance_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) @require_context @@ -955,10 +955,10 @@ def key_pair_destroy_all_by_user(context, user_id): session = get_session() with session.begin(): session.query(models.KeyPair).\ - filter_by(user_id=user_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + filter_by(user_id=user_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) @require_context @@ -1079,8 +1079,8 @@ def network_disassociate(context, network_id): def network_disassociate_all(context): session = get_session() session.query(models.Network).\ - update({'project_id': None, - 'updated_at': literal_column('updated_at')}) + update({'project_id': None, + 'updated_at': literal_column('updated_at')}) @require_context @@ -1463,16 +1463,16 @@ def volume_destroy(context, volume_id): session = get_session() with session.begin(): session.query(models.Volume).\ - filter_by(id=volume_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + filter_by(id=volume_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) session.query(models.ExportDevice).\ - filter_by(volume_id=volume_id).\ - update({'volume_id': None}) + filter_by(volume_id=volume_id).\ + update({'volume_id': None}) session.query(models.IscsiTarget).\ - filter_by(volume_id=volume_id).\ - update({'volume_id': None}) + filter_by(volume_id=volume_id).\ + update({'volume_id': None}) @require_admin_context @@ -1693,20 +1693,20 @@ def security_group_destroy(context, security_group_id): session = get_session() with session.begin(): session.query(models.SecurityGroup).\ - filter_by(id=security_group_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + filter_by(id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupInstanceAssociation).\ - filter_by(security_group_id=security_group_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + filter_by(security_group_id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupIngressRule).\ - filter_by(group_id=security_group_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + filter_by(group_id=security_group_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) @require_context @@ -1715,13 +1715,13 @@ def security_group_destroy_all(context, session=None): session = get_session() with session.begin(): session.query(models.SecurityGroup).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupIngressRule).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), - 'updated_at': literal_column('updated_at')}) + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) ################### @@ -1851,14 +1851,14 @@ def user_delete(context, id): session = get_session() with session.begin(): session.query(models.UserProjectAssociation).\ - filter_by(user_id=id).\ - delete() + filter_by(user_id=id).\ + delete() session.query(models.UserRoleAssociation).\ - filter_by(user_id=id).\ - delete() + filter_by(user_id=id).\ + delete() session.query(models.UserProjectRoleAssociation).\ - filter_by(user_id=id).\ - delete() + filter_by(user_id=id).\ + delete() user_ref = user_get(context, id, session=session) session.delete(user_ref) @@ -1950,11 +1950,11 @@ def project_delete(context, id): session = get_session() with session.begin(): session.query(models.UserProjectAssociation).\ - filter_by(project_id=id).\ - delete() + filter_by(project_id=id).\ + delete() session.query(models.UserProjectRoleAssociation).\ - filter_by(project_id=id).\ - delete() + filter_by(project_id=id).\ + delete() project_ref = project_get(context, id, session=session) session.delete(project_ref) @@ -1980,10 +1980,10 @@ def user_remove_project_role(context, user_id, project_id, role): session = get_session() with session.begin(): session.query(models.UserProjectRoleAssociation).\ - filter_by(user_id=user_id).\ - filter_by(project_id=project_id).\ - filter_by(role=role).\ - delete() + filter_by(user_id=user_id).\ + filter_by(project_id=project_id).\ + filter_by(role=role).\ + delete() def user_remove_role(context, user_id, role): @@ -2135,8 +2135,8 @@ def console_delete(context, console_id): with session.begin(): # consoles are meant to be transient. (mdragon) session.query(models.Console).\ - filter_by(id=console_id).\ - delete() + filter_by(id=console_id).\ + delete() def console_get_by_pool_instance(context, pool_id, instance_id): @@ -2293,8 +2293,8 @@ def zone_delete(context, zone_id): session = get_session() with session.begin(): session.query(models.Zone).\ - filter_by(id=zone_id).\ - delete() + filter_by(id=zone_id).\ + delete() @require_admin_context -- cgit From 0a130010c26e4ef9ba5b9917ff47766de7805ab9 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Fri, 11 Mar 2011 14:24:03 -0500 Subject: process_input for tee. fixes: 733439 --- nova/virt/disk.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/virt/disk.py b/nova/virt/disk.py index 5d499c42c..9abe44cc3 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -189,4 +189,4 @@ def _inject_net_into_fs(net, fs): utils.execute('sudo', 'chown', 'root:root', netdir) utils.execute('sudo', 'chmod', 755, netdir) netfile = os.path.join(netdir, 'interfaces') - utils.execute('sudo', 'tee', netfile, net) + utils.execute('sudo', 'tee', netfile, process_input=net) -- cgit From be9734b03bce871d32e21da2ba341dfa42aa020a Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Fri, 11 Mar 2011 17:19:14 -0500 Subject: Fixed lp732866 by catching relevant `exception.NotFound` exception. Tests did not uncover this vulnerability due to "incorrect" FakeAuthManager. I say "incorrect" because potentially different implementations (LDAP or Database driven) of AuthManager might return different errors from `get_user_from_access_key`. Also, removed all references to 'bacon', 'ham', 'herp', and 'derp' and replaced them with hopefully more helpful terms. --- nova/api/openstack/auth.py | 6 +++- nova/tests/api/openstack/fakes.py | 5 +++- nova/tests/api/openstack/test_auth.py | 56 +++++++++++++++++++++-------------- 3 files changed, 43 insertions(+), 24 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 4c6b58eff..f3a9bdeca 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -135,7 +135,11 @@ class AuthMiddleware(wsgi.Middleware): req - wsgi.Request object """ ctxt = context.get_admin_context() - user = self.auth.get_user_from_access_key(key) + + try: + user = self.auth.get_user_from_access_key(key) + except exception.NotFound: + user = None if user and user.name == username: token_hash = hashlib.sha1('%s%s%f' % (username, key, diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index e50d11a3d..7cb974bb2 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -321,7 +321,10 @@ class FakeAuthManager(object): (user.id == p.project_manager_id)] def get_user_from_access_key(self, key): - return FakeAuthManager.auth_data.get(key, None) + try: + return FakeAuthManager.auth_data[key] + except KeyError: + raise exc.NotFound class FakeRateLimiter(object): diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index aaaa4e415..d1b3c0e10 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -51,11 +51,12 @@ class Test(test.TestCase): def test_authorize_user(self): f = fakes.FakeAuthManager() - f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) + f.add_user('user1_key', + nova.auth.manager.User(1, 'user1', None, None, None)) req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-User'] = 'herp' - req.headers['X-Auth-Key'] = 'derp' + req.headers['X-Auth-User'] = 'user1' + req.headers['X-Auth-Key'] = 'user1_key' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) @@ -65,13 +66,13 @@ class Test(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - u = nova.auth.manager.User(1, 'herp', None, None, None) - f.add_user('derp', u) - f.create_project('test', u) + u = nova.auth.manager.User(1, 'user1', None, None, None) + f.add_user('user1_key', u) + f.create_project('user1_project', u) req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) - req.headers['X-Auth-User'] = 'herp' - req.headers['X-Auth-Key'] = 'derp' + req.headers['X-Auth-User'] = 'user1' + req.headers['X-Auth-Key'] = 'user1_key' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) @@ -92,7 +93,7 @@ class Test(test.TestCase): def test_token_expiry(self): self.destroy_called = False - token_hash = 'bacon' + token_hash = 'token_hash' def destroy_token_mock(meh, context, token): self.destroy_called = True @@ -109,15 +110,26 @@ class Test(test.TestCase): bad_token) req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-Token'] = 'bacon' + req.headers['X-Auth-Token'] = 'token_hash' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') self.assertEqual(self.destroy_called, True) - def test_bad_user(self): + def test_bad_user_bad_key(self): + req = webob.Request.blank('/v1.0/') + req.headers['X-Auth-User'] = 'unknown_user' + req.headers['X-Auth-Key'] = 'unknown_user_key' + result = req.get_response(fakes.wsgi_app()) + self.assertEqual(result.status, '401 Unauthorized') + + def test_bad_user_good_key(self): + f = fakes.FakeAuthManager() + u = nova.auth.manager.User(1, 'user1', None, None, None) + f.add_user('user1_key', u) + req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-User'] = 'herp' - req.headers['X-Auth-Key'] = 'derp' + req.headers['X-Auth-User'] = 'unknown_user' + req.headers['X-Auth-Key'] = 'user1_key' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') @@ -128,7 +140,7 @@ class Test(test.TestCase): def test_bad_token(self): req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-Token'] = 'baconbaconbacon' + req.headers['X-Auth-Token'] = 'unknown_token' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') @@ -137,11 +149,11 @@ class TestFunctional(test.TestCase): def test_token_expiry(self): ctx = context.get_admin_context() tok = db.auth_token_create(ctx, dict( - token_hash='bacon', + token_hash='test_token_hash', cdn_management_url='', server_management_url='', storage_url='', - user_id='ham', + user_id='user1', )) db.auth_token_update(ctx, tok.token_hash, dict( @@ -149,13 +161,13 @@ class TestFunctional(test.TestCase): )) req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-Token'] = 'bacon' + req.headers['X-Auth-Token'] = 'test_token_hash' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') def test_token_doesnotexist(self): req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-Token'] = 'ham' + req.headers['X-Auth-Token'] = 'nonexistant_token_hash' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') @@ -178,13 +190,13 @@ class TestLimiter(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - u = nova.auth.manager.User(1, 'herp', None, None, None) - f.add_user('derp', u) + u = nova.auth.manager.User(1, 'user1', None, None, None) + f.add_user('user1_key', u) f.create_project('test', u) req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-User'] = 'herp' - req.headers['X-Auth-Key'] = 'derp' + req.headers['X-Auth-User'] = 'user1' + req.headers['X-Auth-Key'] = 'user1_key' result = req.get_response(fakes.wsgi_app()) self.assertEqual(len(result.headers['X-Auth-Token']), 40) -- cgit From 2ac7fa75c02c885fc9d4dfacba8318aadbdbfceb Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 11 Mar 2011 23:34:26 +0100 Subject: Indentation adjustment (cosmetical). --- nova/virt/libvirt_conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index d82b33ddd..678331eed 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1243,7 +1243,7 @@ class IptablesFirewallDriver(FirewallDriver): self.iptables.apply() else: LOG.info(_('Attempted to unfilter instance %s which is not ' - 'filtered'), instance['id']) + 'filtered'), instance['id']) def prepare_instance_filter(self, instance): self.instances[instance['id']] = instance -- cgit From a9d71273742f440af5687650dd9cd72d827a6bef Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 11 Mar 2011 23:36:28 +0100 Subject: Make the fallback value None instead of False --- nova/virt/libvirt_conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 678331eed..d2061a0ca 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1238,7 +1238,7 @@ class IptablesFirewallDriver(FirewallDriver): pass def unfilter_instance(self, instance): - if self.instances.pop(instance['id'], False): + if self.instances.pop(instance['id'], None): self.remove_filters_for_instance(instance) self.iptables.apply() else: -- cgit From 909b72faa77ba0a2bc787309b95fdfae9bb9ca01 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Fri, 11 Mar 2011 17:41:10 -0500 Subject: Removed EOL whitespace in accordance with PEP-8. --- nova/tests/api/openstack/test_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index d1b3c0e10..0448ed701 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -51,7 +51,7 @@ class Test(test.TestCase): def test_authorize_user(self): f = fakes.FakeAuthManager() - f.add_user('user1_key', + f.add_user('user1_key', nova.auth.manager.User(1, 'user1', None, None, None)) req = webob.Request.blank('/v1.0/') -- cgit From b3f5a4d5a8e513fe65a3b1dde9b36fd1388afb67 Mon Sep 17 00:00:00 2001 From: "Kevin L. Mitchell" Date: Fri, 11 Mar 2011 22:55:56 +0000 Subject: Remove vish comment --- nova/db/sqlalchemy/api.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'nova') diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 08bc8fe2f..71b85d659 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -578,8 +578,6 @@ def fixed_ip_disassociate(context, address): @require_admin_context def fixed_ip_disassociate_all_by_timeout(_context, host, time): session = get_session() - # NOTE(vish): The nested select is because sqlite doesn't support - # JOINs in UPDATEs. inner_q = session.query(models.Network.id).\ filter_by(host=host).\ subquery() -- cgit From da76b3d67b2c2e864025c4ba201b63e1dee2ff1f Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 16:58:18 -0600 Subject: Review feedback --- nova/virt/xenapi/vm_utils.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'nova') diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index a42cabfe4..866eb5d62 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -101,7 +101,7 @@ class VMHelper(HelperBase): get_instance_type(instance.instance_type) mem = str(long(instance_type['memory_mb']) * 1024 * 1024) vcpus = str(instance_type['vcpus']) - vm_rec = { + rec = { 'actions_after_crash': 'destroy', 'actions_after_reboot': 'restart', 'actions_after_shutdown': 'destroy', @@ -145,21 +145,21 @@ class VMHelper(HelperBase): vm_rec['platform']['nx'] = 'false' if instance.kernel_id: # 1. Kernel explicitly passed in, use that - vm_rec['PV_args'] = 'root=/dev/xvda1' - vm_rec['PV_kernel'] = kernel - vm_rec['PV_ramdisk'] = ramdisk + rec['PV_args'] = 'root=/dev/xvda1' + rec['PV_kernel'] = kernel + rec['PV_ramdisk'] = ramdisk else: # 2. Use kernel within the image - vm_rec['PV_args'] = 'clocksource=jiffies' - vm_rec['PV_bootloader'] = 'pygrub' + rec['PV_args'] = 'clocksource=jiffies' + rec['PV_bootloader'] = 'pygrub' else: # 3. Using hardware virtualization - vm_rec['platform']['nx'] = 'true' - vm_rec['HVM_boot_params'] = {'order': 'dc'} - vm_rec['HVM_boot_policy'] = 'BIOS order' + rec['platform']['nx'] = 'true' + rec['HVM_boot_params'] = {'order': 'dc'} + rec['HVM_boot_policy'] = 'BIOS order' LOG.debug(_('Created VM %s...'), instance.name) - vm_ref = session.call_xenapi('VM.create', vm_rec) + vm_ref = session.call_xenapi('VM.create', rec) instance_name = instance.name LOG.debug(_('Created VM %(instance_name)s as %(vm_ref)s.') % locals()) return vm_ref -- cgit From 80bc32659e41f496bb1bfefbdd6ca63de7ff9f98 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 17:11:25 -0600 Subject: Review feedback --- nova/virt/xenapi/vmops.py | 61 +++++++++++++++++++++++++---------------------- 1 file changed, 33 insertions(+), 28 deletions(-) (limited to 'nova') diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 190c2022d..3ccdf9d80 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -434,7 +434,7 @@ class VMOps(object): raise RuntimeError(resp_dict['message']) return resp_dict['message'] - def _shutdown(self, instance, vm, hard=True): + def _shutdown(self, instance, vm_ref, hard=True): """Shutdown an instance""" state = self.get_info(instance['name'])['state'] if state == power_state.SHUTDOWN: @@ -448,31 +448,33 @@ class VMOps(object): try: task = None if hard: - task = self._session.call_xenapi("Async.VM.hard_shutdown", vm) + task = self._session.call_xenapi("Async.VM.hard_shutdown", + vm_ref) else: - task = self._session.call_xenapi('Async.VM.clean_shutdown', vm) + task = self._session.call_xenapi("Async.VM.clean_shutdown", + vm_ref) self._session.wait_for_task(task, instance.id) except self.XenAPI.Failure, exc: LOG.exception(exc) - def _destroy_vdis(self, instance, vm): - """Destroys all VDIs associated with a VM """ + def _destroy_vdis(self, instance, vm_ref): + """Destroys all VDIs associated with a VM""" instance_id = instance.id LOG.debug(_("Destroying VDIs for Instance %(instance_id)s") % locals()) - vdis = VMHelper.lookup_vm_vdis(self._session, vm) + vdi_refs = VMHelper.lookup_vm_vdis(self._session, vm_ref) - if not vdis: + if not vdi_refs: return - for vdi in vdis: + for vdi_ref in vdi_refs: try: - task = self._session.call_xenapi('Async.VDI.destroy', vdi) + task = self._session.call_xenapi('Async.VDI.destroy', vdi_ref) self._session.wait_for_task(task, instance.id) except self.XenAPI.Failure, exc: LOG.exception(exc) - def _destroy_kernel_ramdisk(self, instance, vm): + def _destroy_kernel_ramdisk(self, instance, vm_ref): """ Three situations can occur: @@ -499,8 +501,8 @@ class VMOps(object): "both" % locals())) # 3. We have both kernel and ramdisk - (kernel, ramdisk) = VMHelper.lookup_kernel_ramdisk( - self._session, vm) + (kernel, ramdisk) = VMHelper.lookup_kernel_ramdisk(self._session, + vm_ref) LOG.debug(_("Removing kernel/ramdisk files")) @@ -511,11 +513,11 @@ class VMOps(object): LOG.debug(_("kernel/ramdisk files removed")) - def _destroy_vm(self, instance, vm): - """Destroys a VM record """ + def _destroy_vm(self, instance, vm_ref): + """Destroys a VM record""" instance_id = instance.id try: - task = self._session.call_xenapi('Async.VM.destroy', vm) + task = self._session.call_xenapi('Async.VM.destroy', vm_ref) self._session.wait_for_task(task, instance_id) except self.XenAPI.Failure, exc: LOG.exception(exc) @@ -596,8 +598,9 @@ class VMOps(object): - spawn a rescue VM (the vm name-label will be instance-N-rescue) """ - rescue_vm = VMHelper.lookup(self._session, instance.name + "-rescue") - if rescue_vm: + rescue_vm_ref = VMHelper.lookup(self._session, + instance.name + "-rescue") + if rescue_vm_ref: raise RuntimeError(_( "Instance is already in Rescue Mode: %s" % instance.name)) @@ -609,8 +612,8 @@ class VMOps(object): self.spawn(instance) rescue_vm_ref = self._get_vm_opaque_ref(instance) - vbd = self._session.get_xenapi().VM.get_VBDs(vm_ref)[0] - vdi_ref = self._session.get_xenapi().VBD.get_record(vbd)["VDI"] + vbd_ref = self._session.get_xenapi().VM.get_VBDs(vm_ref)[0] + vdi_ref = self._session.get_xenapi().VBD.get_record(vbd_ref)["VDI"] vbd_ref = VMHelper.create_vbd(self._session, rescue_vm_ref, vdi_ref, 1, False) @@ -623,35 +626,37 @@ class VMOps(object): - release the bootlock to allow the instance VM to start """ - rescue_vm = VMHelper.lookup(self._session, instance.name + "-rescue") + rescue_vm_ref = VMHelper.lookup(self._session, + instance.name + "-rescue") - if not rescue_vm: + if not rescue_vm_ref: raise exception.NotFound(_( "Instance is not in Rescue Mode: %s" % instance.name)) original_vm_ref = self._get_vm_opaque_ref(instance) - vbds = self._session.get_xenapi().VM.get_VBDs(rescue_vm) + vbd_refs = self._session.get_xenapi().VM.get_VBDs(rescue_vm_ref) instance._rescue = False - for vbd_ref in vbds: + for vbd_ref in vbd_refs: vbd = self._session.get_xenapi().VBD.get_record(vbd_ref) if vbd["userdevice"] == "1": VMHelper.unplug_vbd(self._session, vbd_ref) VMHelper.destroy_vbd(self._session, vbd_ref) - task1 = self._session.call_xenapi("Async.VM.hard_shutdown", rescue_vm) + task1 = self._session.call_xenapi("Async.VM.hard_shutdown", + rescue_vm_ref) self._session.wait_for_task(task1, instance.id) - vdis = VMHelper.lookup_vm_vdis(self._session, rescue_vm) - for vdi in vdis: + vdi_refs = VMHelper.lookup_vm_vdis(self._session, rescue_vm_ref) + for vdi_ref in vdi_refs: try: - task = self._session.call_xenapi('Async.VDI.destroy', vdi) + task = self._session.call_xenapi('Async.VDI.destroy', vdi_ref) self._session.wait_for_task(task, instance.id) except self.XenAPI.Failure: continue - task2 = self._session.call_xenapi('Async.VM.destroy', rescue_vm) + task2 = self._session.call_xenapi('Async.VM.destroy', rescue_vm_ref) self._session.wait_for_task(task2, instance.id) self._release_bootlock(original_vm_ref) -- cgit From ab37248cc2e40b06e1d349833da01494a9ca3641 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 17:13:10 -0600 Subject: oops --- nova/virt/xenapi/vm_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 866eb5d62..8dd246178 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -142,7 +142,7 @@ class VMHelper(HelperBase): # Complete VM configuration record according to the image type # non-raw/raw with PV kernel/raw in HVM mode if use_pv_kernel: - vm_rec['platform']['nx'] = 'false' + rec['platform']['nx'] = 'false' if instance.kernel_id: # 1. Kernel explicitly passed in, use that rec['PV_args'] = 'root=/dev/xvda1' -- cgit From 29e59a96602265c5b5746865db94a3f00b8b5cf5 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 11 Mar 2011 15:21:31 -0800 Subject: Change capitalization of Openstack to OpenStack --- nova/tests/integrated/api/client.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'nova') diff --git a/nova/tests/integrated/api/client.py b/nova/tests/integrated/api/client.py index da8d87e07..245eb8c69 100644 --- a/nova/tests/integrated/api/client.py +++ b/nova/tests/integrated/api/client.py @@ -24,7 +24,7 @@ from nova import log as logging LOG = logging.getLogger('nova.tests.api') -class OpenstackApiException(Exception): +class OpenStackApiException(Exception): def __init__(self, message=None, response=None): self.response = response if not message: @@ -37,22 +37,22 @@ class OpenstackApiException(Exception): message = _('%(message)s\nStatus Code: %(_status)s\n' 'Body: %(_body)s') % locals() - super(OpenstackApiException, self).__init__(message) + super(OpenStackApiException, self).__init__(message) -class OpenstackApiAuthenticationException(OpenstackApiException): +class OpenStackApiAuthenticationException(OpenStackApiException): def __init__(self, response=None, message=None): if not message: message = _("Authentication error") - super(OpenstackApiAuthenticationException, self).__init__(message, + super(OpenStackApiAuthenticationException, self).__init__(message, response) -class OpenstackApiNotFoundException(OpenstackApiException): +class OpenStackApiNotFoundException(OpenStackApiException): def __init__(self, response=None, message=None): if not message: message = _("Item not found") - super(OpenstackApiNotFoundException, self).__init__(message, response) + super(OpenStackApiNotFoundException, self).__init__(message, response) class TestOpenStackClient(object): @@ -82,7 +82,7 @@ class TestOpenStackClient(object): conn = httplib.HTTPSConnection(hostname, port=port) else: - raise OpenstackApiException("Unknown scheme: %s" % url) + raise OpenStackApiException("Unknown scheme: %s" % url) relative_url = parsed_url.path if parsed_url.query: @@ -111,7 +111,7 @@ class TestOpenStackClient(object): # Until bug732866 is fixed, we can't check this properly... #if http_status == 401: if http_status != 204: - raise OpenstackApiAuthenticationException(response=response) + raise OpenStackApiAuthenticationException(response=response) auth_headers = {} for k, v in response.getheaders(): @@ -138,9 +138,9 @@ class TestOpenStackClient(object): if check_response_status: if not http_status in check_response_status: if http_status == 404: - raise OpenstackApiNotFoundException(response=response) + raise OpenStackApiNotFoundException(response=response) else: - raise OpenstackApiException( + raise OpenStackApiException( message=_("Unexpected status code"), response=response) -- cgit From f9706b5080786a4d3e530f3e8bdb69147e9f5086 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 17:35:37 -0600 Subject: Review feedback --- nova/virt/xenapi/vm_utils.py | 31 ++++++++++++++++--------------- nova/virt/xenapi/vmops.py | 6 +++--- 2 files changed, 19 insertions(+), 18 deletions(-) (limited to 'nova') diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 8dd246178..7aa3f3c3b 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -633,37 +633,38 @@ class VMHelper(HelperBase): return is_pv @classmethod - def lookup(cls, session, i): + def lookup(cls, session, name_label): """Look the instance i up, and returns it if available""" - vms = session.get_xenapi().VM.get_by_name_label(i) - n = len(vms) + vm_refs = session.get_xenapi().VM.get_by_name_label(name_label) + n = len(vm_refs) if n == 0: return None elif n > 1: - raise exception.Duplicate(_('duplicate name found: %s') % i) + raise exception.Duplicate(_('duplicate name found: %s') % + name_label) else: - return vms[0] + return vm_refs[0] @classmethod - def lookup_vm_vdis(cls, session, vm): + def lookup_vm_vdis(cls, session, vm_ref): """Look for the VDIs that are attached to the VM""" # Firstly we get the VBDs, then the VDIs. # TODO(Armando): do we leave the read-only devices? - vbds = session.get_xenapi().VM.get_VBDs(vm) - vdis = [] - if vbds: - for vbd in vbds: + vbd_refs = session.get_xenapi().VM.get_VBDs(vm_ref) + vdi_refs = [] + if vbd_refs: + for vbd_ref in vbd_refs: try: - vdi = session.get_xenapi().VBD.get_VDI(vbd) + vdi_ref = session.get_xenapi().VBD.get_VDI(vbd_ref) # Test valid VDI - record = session.get_xenapi().VDI.get_record(vdi) + record = session.get_xenapi().VDI.get_record(vdi_ref) LOG.debug(_('VDI %s is still available'), record['uuid']) except cls.XenAPI.Failure, exc: LOG.exception(exc) else: - vdis.append(vdi) - if len(vdis) > 0: - return vdis + vdi_refs.append(vdi_ref) + if len(vdi_refs) > 0: + return vdi_refs else: return None diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 3ccdf9d80..0faec1169 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -614,10 +614,10 @@ class VMOps(object): vbd_ref = self._session.get_xenapi().VM.get_VBDs(vm_ref)[0] vdi_ref = self._session.get_xenapi().VBD.get_record(vbd_ref)["VDI"] - vbd_ref = VMHelper.create_vbd(self._session, rescue_vm_ref, vdi_ref, - 1, False) + rescue_vbd_ref = VMHelper.create_vbd(self._session, rescue_vm_ref, + vdi_ref, 1, False) - self._session.call_xenapi("Async.VBD.plug", vbd_ref) + self._session.call_xenapi("Async.VBD.plug", rescue_vbd_ref) def unrescue(self, instance, callback): """Unrescue the specified instance -- cgit From cdd8790426d3eb77712f5a19f99211b12a9ad9c5 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 17:48:44 -0600 Subject: Review feedback --- nova/virt/xenapi/vm_utils.py | 10 +++++----- nova/virt/xenapi/vmops.py | 6 +++--- nova/virt/xenapi/volume_utils.py | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) (limited to 'nova') diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 7aa3f3c3b..1a872345d 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -202,13 +202,13 @@ class VMHelper(HelperBase): @classmethod def find_vbd_by_number(cls, session, vm_ref, number): """Get the VBD reference from the device number""" - vbds = session.get_xenapi().VM.get_VBDs(vm_ref) - if vbds: - for vbd in vbds: + vbd_refs = session.get_xenapi().VM.get_VBDs(vm_ref) + if vbd_refs: + for vbd_ref in vbd_refs: try: - vbd_rec = session.get_xenapi().VBD.get_record(vbd) + vbd_rec = session.get_xenapi().VBD.get_record(vbd_ref) if vbd_rec['userdevice'] == str(number): - return vbd + return vbd_ref except cls.XenAPI.Failure, exc: LOG.exception(exc) raise StorageError(_('VBD not found in instance %s') % vm_ref) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 0faec1169..382915b0c 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -55,12 +55,12 @@ class VMOps(object): def list_instances(self): """List VM instances""" - vms = [] + vm_refs = [] for vm_ref in self._session.get_xenapi().VM.get_all(): vm_rec = self._session.get_xenapi().VM.get_record(vm_ref) if not vm_rec["is_a_template"] and not vm_rec["is_control_domain"]: - vms.append(vm_rec["name_label"]) - return vms + vm_refs.append(vm_rec["name_label"]) + return vm_refs def _start(self, instance, vm_ref=None): """Power on a VM instance""" diff --git a/nova/virt/xenapi/volume_utils.py b/nova/virt/xenapi/volume_utils.py index d5ebd29d5..72284ac02 100644 --- a/nova/virt/xenapi/volume_utils.py +++ b/nova/virt/xenapi/volume_utils.py @@ -117,16 +117,16 @@ class VolumeHelper(HelperBase): def introduce_vdi(cls, session, sr_ref): """Introduce VDI in the host""" try: - vdis = session.get_xenapi().SR.get_VDIs(sr_ref) + vdi_refs = session.get_xenapi().SR.get_VDIs(sr_ref) except cls.XenAPI.Failure, exc: LOG.exception(exc) raise StorageError(_('Unable to introduce VDI on SR %s') % sr_ref) try: - vdi_rec = session.get_xenapi().VDI.get_record(vdis[0]) + vdi_rec = session.get_xenapi().VDI.get_record(vdi_refs[0]) except cls.XenAPI.Failure, exc: LOG.exception(exc) raise StorageError(_('Unable to get record' - ' of VDI %s on') % vdis[0]) + ' of VDI %s on') % vdi_refs[0]) else: try: return session.get_xenapi().VDI.introduce( -- cgit From 48196abaf7e9c47bfda3f744e0be9bc242004b72 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 18:00:34 -0600 Subject: Review feedback --- nova/virt/xenapi/vm_utils.py | 56 ++++++++++++++++++++++---------------------- nova/virt/xenapi/vmops.py | 8 +++---- 2 files changed, 32 insertions(+), 32 deletions(-) (limited to 'nova') diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 1a872345d..4d55937e3 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -443,29 +443,29 @@ class VMHelper(HelperBase): vdi_size += MBR_SIZE_BYTES name_label = get_name_label_for_image(image) - vdi = cls.create_vdi(session, sr_ref, name_label, vdi_size, False) + vdi_ref = cls.create_vdi(session, sr_ref, name_label, vdi_size, False) - with_vdi_attached_here(session, vdi, False, + with_vdi_attached_here(session, vdi_ref, False, lambda dev: _stream_disk(dev, image_type, virtual_size, image_file)) if image_type == ImageType.KERNEL_RAMDISK: #we need to invoke a plugin for copying VDI's #content into proper path - LOG.debug(_("Copying VDI %s to /boot/guest on dom0"), vdi) + LOG.debug(_("Copying VDI %s to /boot/guest on dom0"), vdi_ref) fn = "copy_kernel_vdi" args = {} - args['vdi-ref'] = vdi + args['vdi-ref'] = vdi_ref #let the plugin copy the correct number of bytes args['image-size'] = str(vdi_size) task = session.async_call_plugin('glance', fn, args) filename = session.wait_for_task(task, instance_id) #remove the VDI as it is not needed anymore - session.get_xenapi().VDI.destroy(vdi) - LOG.debug(_("Kernel/Ramdisk VDI %s destroyed"), vdi) + session.get_xenapi().VDI.destroy(vdi_ref) + LOG.debug(_("Kernel/Ramdisk VDI %s destroyed"), vdi_ref) return filename else: - return session.get_xenapi().VDI.get_uuid(vdi) + return session.get_xenapi().VDI.get_uuid(vdi_ref) @classmethod def determine_disk_image_type(cls, instance): @@ -840,16 +840,16 @@ def safe_find_sr(session): def find_sr(session): """Return the storage repository to hold VM images""" host = session.get_xenapi_host() - srs = session.get_xenapi().SR.get_all() - for sr in srs: - sr_rec = session.get_xenapi().SR.get_record(sr) + sr_refs = session.get_xenapi().SR.get_all() + for sr_ref in sr_refs: + sr_rec = session.get_xenapi().SR.get_record(sr_ref) if not ('i18n-key' in sr_rec['other_config'] and sr_rec['other_config']['i18n-key'] == 'local-storage'): continue - for pbd in sr_rec['PBDs']: - pbd_rec = session.get_xenapi().PBD.get_record(pbd) + for pbd_ref in sr_rec['PBDs']: + pbd_rec = session.get_xenapi().PBD.get_record(pbd_ref) if pbd_rec['host'] == host: - return sr + return sr_ref return None @@ -874,11 +874,11 @@ def remap_vbd_dev(dev): return remapped_dev -def with_vdi_attached_here(session, vdi, read_only, f): +def with_vdi_attached_here(session, vdi_ref, read_only, f): this_vm_ref = get_this_vm_ref(session) vbd_rec = {} vbd_rec['VM'] = this_vm_ref - vbd_rec['VDI'] = vdi + vbd_rec['VDI'] = vdi_ref vbd_rec['userdevice'] = 'autodetect' vbd_rec['bootable'] = False vbd_rec['mode'] = read_only and 'RO' or 'RW' @@ -889,14 +889,14 @@ def with_vdi_attached_here(session, vdi, read_only, f): vbd_rec['qos_algorithm_type'] = '' vbd_rec['qos_algorithm_params'] = {} vbd_rec['qos_supported_algorithms'] = [] - LOG.debug(_('Creating VBD for VDI %s ... '), vdi) - vbd = session.get_xenapi().VBD.create(vbd_rec) - LOG.debug(_('Creating VBD for VDI %s done.'), vdi) + LOG.debug(_('Creating VBD for VDI %s ... '), vdi_ref) + vbd_ref = session.get_xenapi().VBD.create(vbd_rec) + LOG.debug(_('Creating VBD for VDI %s done.'), vdi_ref) try: - LOG.debug(_('Plugging VBD %s ... '), vbd) - session.get_xenapi().VBD.plug(vbd) - LOG.debug(_('Plugging VBD %s done.'), vbd) - orig_dev = session.get_xenapi().VBD.get_device(vbd) + LOG.debug(_('Plugging VBD %s ... '), vbd_ref) + session.get_xenapi().VBD.plug(vbd_ref) + LOG.debug(_('Plugging VBD %s done.'), vbd_ref) + orig_dev = session.get_xenapi().VBD.get_device(vbd_ref) LOG.debug(_('VBD %(vbd)s plugged as %(orig_dev)s') % locals()) dev = remap_vbd_dev(orig_dev) if dev != orig_dev: @@ -904,13 +904,13 @@ def with_vdi_attached_here(session, vdi, read_only, f): 'remapping to %(dev)s') % locals()) return f(dev) finally: - LOG.debug(_('Destroying VBD for VDI %s ... '), vdi) - vbd_unplug_with_retry(session, vbd) - ignore_failure(session.get_xenapi().VBD.destroy, vbd) - LOG.debug(_('Destroying VBD for VDI %s done.'), vdi) + LOG.debug(_('Destroying VBD for VDI %s ... '), vdi_ref) + vbd_unplug_with_retry(session, vbd_ref) + ignore_failure(session.get_xenapi().VBD.destroy, vbd_ref) + LOG.debug(_('Destroying VBD for VDI %s done.'), vdi_ref) -def vbd_unplug_with_retry(session, vbd): +def vbd_unplug_with_retry(session, vbd_ref): """Call VBD.unplug on the given VBD, with a retry if we get DEVICE_DETACH_REJECTED. For reasons which I don't understand, we're seeing the device still in use, even when all processes using the device @@ -918,7 +918,7 @@ def vbd_unplug_with_retry(session, vbd): # FIXME(sirp): We can use LoopingCall here w/o blocking sleep() while True: try: - session.get_xenapi().VBD.unplug(vbd) + session.get_xenapi().VBD.unplug(vbd_ref) LOG.debug(_('VBD.unplug successful first time.')) return except VMHelper.XenAPI.Failure, e: diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 382915b0c..fcb290d03 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -87,8 +87,8 @@ class VMOps(object): def _spawn_with_disk(self, instance, vdi_uuid): """Create VM instance""" instance_name = instance.name - vm = VMHelper.lookup(self._session, instance_name) - if vm is not None: + vm_ref = VMHelper.lookup(self._session, instance_name) + if vm_ref is not None: raise exception.Duplicate(_('Attempted to create' ' non-unique name %s') % instance_name) @@ -639,8 +639,8 @@ class VMOps(object): instance._rescue = False for vbd_ref in vbd_refs: - vbd = self._session.get_xenapi().VBD.get_record(vbd_ref) - if vbd["userdevice"] == "1": + _vbd_ref = self._session.get_xenapi().VBD.get_record(vbd_ref) + if _vbd_ref["userdevice"] == "1": VMHelper.unplug_vbd(self._session, vbd_ref) VMHelper.destroy_vbd(self._session, vbd_ref) -- cgit From b944cbcbf023ca321edcc511354b56aa5b07d438 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Fri, 11 Mar 2011 18:03:19 -0600 Subject: oops --- nova/virt/xenapi/vm_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'nova') diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 4d55937e3..f07b57796 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -897,10 +897,10 @@ def with_vdi_attached_here(session, vdi_ref, read_only, f): session.get_xenapi().VBD.plug(vbd_ref) LOG.debug(_('Plugging VBD %s done.'), vbd_ref) orig_dev = session.get_xenapi().VBD.get_device(vbd_ref) - LOG.debug(_('VBD %(vbd)s plugged as %(orig_dev)s') % locals()) + LOG.debug(_('VBD %(vbd_ref)s plugged as %(orig_dev)s') % locals()) dev = remap_vbd_dev(orig_dev) if dev != orig_dev: - LOG.debug(_('VBD %(vbd)s plugged into wrong dev, ' + LOG.debug(_('VBD %(vbd_ref)s plugged into wrong dev, ' 'remapping to %(dev)s') % locals()) return f(dev) finally: -- cgit From c8fc7ed48be84e3b39ab88c8c103fbe52b6718e1 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 14 Mar 2011 14:06:10 +0100 Subject: Add a unit test --- nova/network/linux_net.py | 17 +++++++++++------ nova/tests/test_network.py | 27 +++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) (limited to 'nova') diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 0bcc36081..f55662a7a 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -18,7 +18,7 @@ Implements vlans, bridges, and iptables rules using linux utilities. """ import os -import time +import calendar from nova import db from nova import exception @@ -380,12 +380,17 @@ interface %s def _host_lease(fixed_ip_ref): """Return a host string for an address in leasefile format""" instance_ref = fixed_ip_ref['instance'] - timestamp = time.mktime(instance_ref['updated_at'].timetuple()) + if instance_ref['updated_at']: + timestamp = instance_ref['updated_at'] + else: + timestamp = instance_ref['created_at'] + + seconds_since_epoch = calendar.timegm(timestamp.utctimetuple()) - return "%d %s %s %s" % (timestamp + FLAGS.dhcp_lease_time, - instance_ref['mac_address'], - instance_ref['hostname'], - fixed_ip_ref['address']) + return "%d %s %s %s *" % (seconds_since_epoch + FLAGS.dhcp_lease_time, + instance_ref['mac_address'], + fixed_ip_ref['address'], + instance_ref['hostname'] or '*') def _host_dhcp(fixed_ip_ref): diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index ce1c77210..b7a76be85 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -20,6 +20,7 @@ Unit Tests for network code """ import IPy import os +import time from nova import context from nova import db @@ -29,6 +30,7 @@ from nova import log as logging from nova import test from nova import utils from nova.auth import manager +from nova.network import linux_net FLAGS = flags.FLAGS LOG = logging.getLogger('nova.tests.network') @@ -321,6 +323,31 @@ class NetworkTestCase(test.TestCase): network['id']) self.assertEqual(ip_count, num_available_ips) + def test_dhcp_lease_output(self): + admin_ctxt = context.get_admin_context() + address = self._create_address(0, self.instance_id) + lease_ip(address) + network_ref = db.network_get_by_instance(admin_ctxt, self.instance_id) + leases = linux_net.get_dhcp_leases(context.get_admin_context(), + network_ref['id']) + for line in leases.split('\n'): + seconds, mac, ip, hostname, client_id = line.split(' ') + self.assertTrue(int(seconds) > time.time(), 'Lease expires in ' + 'the past') + octets = mac.split(':') + self.assertEqual(len(octets), 6, "Wrong number of octets " + "in %s" % (max,)) + for octet in octets: + self.assertEqual(len(octet), 2, "Oddly sized octet: %s" + % (octet,)) + # This will throw an exception if the octet is invalid + int(octet, 16) + + # And this will raise an exception in case of an invalid IP + IPy.IP(ip) + + release_ip(address) + def is_allocated_in_project(address, project_id): """Returns true if address is in specified project""" -- cgit From ce57cff740bc7f821bdbb0dd1367c037b6fa1c01 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 14:02:27 -0700 Subject: The exception is called "ApiError", not "APIError" --- nova/virt/libvirt_conn.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 0b306c950..7994e9547 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -362,19 +362,19 @@ class LibvirtConnection(object): @exception.wrap_exception def pause(self, instance, callback): - raise exception.APIError("pause not supported for libvirt.") + raise exception.ApiError("pause not supported for libvirt.") @exception.wrap_exception def unpause(self, instance, callback): - raise exception.APIError("unpause not supported for libvirt.") + raise exception.ApiError("unpause not supported for libvirt.") @exception.wrap_exception def suspend(self, instance, callback): - raise exception.APIError("suspend not supported for libvirt") + raise exception.ApiError("suspend not supported for libvirt") @exception.wrap_exception def resume(self, instance, callback): - raise exception.APIError("resume not supported for libvirt") + raise exception.ApiError("resume not supported for libvirt") @exception.wrap_exception def rescue(self, instance, callback=None): @@ -779,7 +779,7 @@ class LibvirtConnection(object): 'cpu_time': cpu_time} def get_diagnostics(self, instance_name): - raise exception.APIError(_("diagnostics are not supported " + raise exception.ApiError(_("diagnostics are not supported " "for libvirt")) def get_disks(self, instance_name): -- cgit From 337bda95a9e12d395f838e81e279c875b056aba9 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 14 Mar 2011 22:17:14 +0100 Subject: Add missing fallback chain for ipv6. --- nova/virt/libvirt_conn.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 0b306c950..03f046cbd 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1597,6 +1597,9 @@ class IptablesFirewallDriver(FirewallDriver): self.iptables.ipv4['filter'].add_chain('sg-fallback') self.iptables.ipv4['filter'].add_rule('sg-fallback', '-j DROP') + if FLAGS.use_ipv6: + self.iptables.ipv6['filter'].add_chain('sg-fallback') + self.iptables.ipv6['filter'].add_rule('sg-fallback', '-j DROP') def setup_basic_filtering(self, instance): """Use NWFilter from libvirt for this.""" -- cgit From 408a2591d60f5d238e60e4be9197ccc7262f2406 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 14 Mar 2011 16:21:33 -0500 Subject: PEP8 cleanup --- nova/tests/db/fakes.py | 4 ++-- nova/tests/test_xenapi.py | 11 +++++------ nova/virt/xenapi/vm_utils.py | 3 +-- 3 files changed, 8 insertions(+), 10 deletions(-) (limited to 'nova') diff --git a/nova/tests/db/fakes.py b/nova/tests/db/fakes.py index 142f6b1c6..5e9a3aa3b 100644 --- a/nova/tests/db/fakes.py +++ b/nova/tests/db/fakes.py @@ -77,8 +77,8 @@ def stub_out_db_instance_api(stubs): 'mac_address': values['mac_address'], 'vcpus': type_data['vcpus'], 'local_gb': type_data['local_gb'], - 'os_type': values['os_type'] - } + 'os_type': values['os_type']} + return FakeModel(base_options) def fake_network_get_by_instance(context, instance_id): diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index cd125a301..8b0affd5c 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -78,8 +78,7 @@ class XenAPIVolumeTestCase(test.TestCase): 'ramdisk_id': 3, 'instance_type': 'm1.large', 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': 'linux' - } + 'os_type': 'linux'} def _create_volume(self, size='0'): """Create a volume object.""" @@ -315,8 +314,8 @@ class XenAPIVMTestCase(test.TestCase): 'ramdisk_id': ramdisk_id, 'instance_type': instance_type, 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': os_type - } + 'os_type': os_type} + conn = xenapi_conn.get_connection(False) instance = db.instance_create(values) conn.spawn(instance) @@ -440,8 +439,8 @@ class XenAPIMigrateInstance(test.TestCase): 'ramdisk_id': None, 'instance_type': 'm1.large', 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': 'linux' - } + 'os_type': 'linux'} + stubs.stub_out_migration_methods(self.stubs) glance_stubs.stubout_glance_client(self.stubs, glance_stubs.FakeGlance) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index f07b57796..763c5fe40 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -136,8 +136,7 @@ class VMHelper(HelperBase): 'VCPUs_at_startup': vcpus, 'VCPUs_max': vcpus, 'VCPUs_params': {}, - 'xenstore_data': {} - } + 'xenstore_data': {}} # Complete VM configuration record according to the image type # non-raw/raw with PV kernel/raw in HVM mode -- cgit From 7c6aa5eb302637dde0d800f7155235858bbabbeb Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 14:32:15 -0700 Subject: Removed duplicated test, renamed same-named (but non-identical) tests --- nova/tests/test_compute.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) (limited to 'nova') diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index e486050be..14559d1dc 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -115,7 +115,7 @@ class ComputeTestCase(test.TestCase): finally: db.instance_destroy(self.context, ref[0]['id']) - def test_create_instance_associates_security_groups(self): + def test_create_instance_associates_security_groups_1(self): """Make sure create associates security groups""" group = self._create_group() instance_ref = models.Instance() @@ -124,18 +124,7 @@ class ComputeTestCase(test.TestCase): instance_ref['hostname'] = 'i-00000001' return instance_ref - def test_create_instance_defaults_display_name(self): - """Verify that an instance cannot be created without a display_name.""" - cases = [dict(), dict(display_name=None)] - for instance in cases: - ref = self.compute_api.create(self.context, - FLAGS.default_instance_type, None, **instance) - try: - self.assertNotEqual(ref[0]['display_name'], None) - finally: - db.instance_destroy(self.context, ref[0]['id']) - - def test_create_instance_associates_security_groups(self): + def test_create_instance_associates_security_groups_2(self): """Make sure create associates security groups""" group = self._create_group() ref = self.compute_api.create( -- cgit From 567e3bc3a7e66896482d83420190a7c4a8df1e5a Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 15:17:00 -0700 Subject: So the first of those tests doesn't pass. Removing as it looks like it was meant to be deleted. --- nova/tests/test_compute.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) (limited to 'nova') diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 14559d1dc..3651f4cef 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -115,16 +115,7 @@ class ComputeTestCase(test.TestCase): finally: db.instance_destroy(self.context, ref[0]['id']) - def test_create_instance_associates_security_groups_1(self): - """Make sure create associates security groups""" - group = self._create_group() - instance_ref = models.Instance() - instance_ref['id'] = 1 - instance_ref['volumes'] = [{'id': 1}, {'id': 2}] - instance_ref['hostname'] = 'i-00000001' - return instance_ref - - def test_create_instance_associates_security_groups_2(self): + def test_create_instance_associates_security_groups(self): """Make sure create associates security groups""" group = self._create_group() ref = self.compute_api.create( -- cgit From a4e94971b696681a5ced189d8f4263c8f77cc531 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 19:57:30 -0700 Subject: Make key_pair optional with OpenStack API --- nova/api/openstack/servers.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index dc28a0782..47ed254ec 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -146,10 +146,14 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) context = req.environ['nova.context'] + + key_name = None + key_data = None key_pairs = auth_manager.AuthManager.get_key_pairs(context) - if not key_pairs: - raise exception.NotFound(_("No keypairs defined")) - key_pair = key_pairs[0] + if key_pairs: + key_pair = key_pairs[0] + key_name = key_pair['name'] + key_data = key_pair['public_key'] image_id = common.get_image_id_from_image_hash(self._image_service, context, env['server']['imageId']) @@ -174,8 +178,8 @@ class Controller(wsgi.Controller): ramdisk_id=ramdisk_id, display_name=env['server']['name'], display_description=env['server']['name'], - key_name=key_pair['name'], - key_data=key_pair['public_key'], + key_name=key_name, + key_data=key_data, metadata=metadata, onset_files=env.get('onset_files', [])) -- cgit From da605eb84f7d5de741225ff936447db01690a04f Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 20:48:33 -0700 Subject: Don't generate insecure passwords where it's easy to use urandom instead --- nova/console/manager.py | 2 +- nova/console/xvp.py | 4 ---- nova/utils.py | 15 ++++++++++----- 3 files changed, 11 insertions(+), 10 deletions(-) (limited to 'nova') diff --git a/nova/console/manager.py b/nova/console/manager.py index 57c75cf4f..bfa571ea9 100644 --- a/nova/console/manager.py +++ b/nova/console/manager.py @@ -69,7 +69,7 @@ class ConsoleProxyManager(manager.Manager): except exception.NotFound: logging.debug(_("Adding console")) if not password: - password = self.driver.generate_password() + password = utils.generate_password(8) if not port: port = self.driver.get_port(context) console_data = {'instance_name': name, diff --git a/nova/console/xvp.py b/nova/console/xvp.py index 68d8c8565..0cedfbb13 100644 --- a/nova/console/xvp.py +++ b/nova/console/xvp.py @@ -91,10 +91,6 @@ class XVPConsoleProxy(object): """Trim password to length, and encode""" return self._xvp_encrypt(password) - def generate_password(self, length=8): - """Returns random console password""" - return os.urandom(length * 2).encode('base64')[:length] - def _rebuild_xvp_conf(self, context): logging.debug(_("Rebuilding xvp conf")) pools = [pool for pool in diff --git a/nova/utils.py b/nova/utils.py index 87e726394..9c8b27d56 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -263,12 +263,17 @@ def generate_mac(): def generate_password(length=20): - """Generate a random sequence of letters and digits - to be used as a password. Note that this is not intended - to represent the ultimate in security. + """Generate a random alphanumeric password, avoiding 'confusing' O,0,I,1. + + Believed to be reasonably secure (with a reasonable password length!) """ - chrs = string.letters + string.digits - return "".join([random.choice(chrs) for i in xrange(length)]) + # 26 letters, 10 digits = 36 + # Remove O, 0, I, 1 => 32 digits + # 32 digits means we're just using the low 5 bit of each byte + chrs = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789" + + random_bytes = os.urandom(length) + return "".join([chrs[ord(random_bytes[i]) % 32] for i in xrange(length)]) def last_octet(address): -- cgit From 3d0cde272e3227978c5875c811c93e1e3df692ed Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 21:01:48 -0700 Subject: Clarify the logic in using 32 symbols --- nova/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/utils.py b/nova/utils.py index 9c8b27d56..0510c3cbe 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -267,9 +267,10 @@ def generate_password(length=20): Believed to be reasonably secure (with a reasonable password length!) """ - # 26 letters, 10 digits = 36 - # Remove O, 0, I, 1 => 32 digits - # 32 digits means we're just using the low 5 bit of each byte + # 26 letters, 10 digits = 36 choices + # Remove O, 0, I, 1 => 32 choices + # 32 choices means we're just using the low 5 bit of each byte, + # so there's no bias introduced by using a modulo chrs = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789" random_bytes = os.urandom(length) -- cgit From 8a41046dc7cafb19afb6719866b11681daaa9082 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 15 Mar 2011 09:48:21 +0100 Subject: Always put the ipv6 fallback in place. FLAGS.use_ipv6 does not exist yet when the firewall driver is instantiated and the iptables manager takes care not to fiddle with ipv6 if not enabled. --- nova/virt/libvirt_conn.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 03f046cbd..f87decaa0 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1597,9 +1597,8 @@ class IptablesFirewallDriver(FirewallDriver): self.iptables.ipv4['filter'].add_chain('sg-fallback') self.iptables.ipv4['filter'].add_rule('sg-fallback', '-j DROP') - if FLAGS.use_ipv6: - self.iptables.ipv6['filter'].add_chain('sg-fallback') - self.iptables.ipv6['filter'].add_rule('sg-fallback', '-j DROP') + self.iptables.ipv6['filter'].add_chain('sg-fallback') + self.iptables.ipv6['filter'].add_rule('sg-fallback', '-j DROP') def setup_basic_filtering(self, instance): """Use NWFilter from libvirt for this.""" -- cgit From 56ff68056254610c4f0eb5cd5c5432a68ed30b2f Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 15 Mar 2011 10:42:32 -0700 Subject: Support testing the OpenStack API without key_pairs --- nova/tests/api/openstack/fakes.py | 11 +++++++++-- nova/tests/api/openstack/test_servers.py | 9 ++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index e50d11a3d..ccc853360 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -85,10 +85,17 @@ def wsgi_app(inner_application=None): return mapper -def stub_out_key_pair_funcs(stubs): +def stub_out_key_pair_funcs(stubs, have_key_pair=True): def key_pair(context, user_id): return [dict(name='key', public_key='public_key')] - stubs.Set(nova.db, 'key_pair_get_all_by_user', key_pair) + + def no_key_pair(context, user_id): + return [] + + if have_key_pair: + stubs.Set(nova.db, 'key_pair_get_all_by_user', key_pair) + else: + stubs.Set(nova.db, 'key_pair_get_all_by_user', no_key_pair) def stub_out_image_service(stubs): diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 5d7a208e9..40026a615 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -216,7 +216,7 @@ class ServersTest(test.TestCase): servers = json.loads(res.body)['servers'] self.assertEqual([s['id'] for s in servers], [1, 2]) - def test_create_instance(self): + def _test_create_instance_helper(self, with_key_pair): def instance_create(context, inst): return {'id': '1', 'display_name': 'server_test'} @@ -271,6 +271,13 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) + def test_create_instance(self): + self._test_create_instance_helper(True) + + def test_create_instance_no_key_pair(self): + fakes.stub_out_key_pair_funcs(self.stubs, False) + self._test_create_instance_helper(False) + def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' -- cgit From 22aad6700124411aceed0b2bd3953cbbc48b6130 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 15 Mar 2011 11:24:07 -0700 Subject: Use random.SystemRandom for easy secure randoms, configurable symbol set by default including mixed-case --- nova/utils.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) (limited to 'nova') diff --git a/nova/utils.py b/nova/utils.py index 0510c3cbe..199ee8701 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -262,19 +262,25 @@ def generate_mac(): return ':'.join(map(lambda x: "%02x" % x, mac)) -def generate_password(length=20): - """Generate a random alphanumeric password, avoiding 'confusing' O,0,I,1. +# Default symbols to use for passwords. Avoids visually confusing characters. +# ~6 bits per symbol +DEFAULT_PASSWORD_SYMBOLS = ("23456789" # Removed: 0,1 + "ABCDEFGHJKLMNPQRSTUVWXYZ" # Removed: I, O + "abcdefghijkmnopqrstuvwxyz") # Removed: l + + +# ~5 bits per symbol +EASIER_PASSWORD_SYMBOLS = ("23456789" # Removed: 0, 1 + "ABCDEFGHJKLMNPQRSTUVWXYZ") # Removed: I, O + + +def generate_password(length=20, symbols=DEFAULT_PASSWORD_SYMBOLS): + """Generate a random password from the supplied symbols. Believed to be reasonably secure (with a reasonable password length!) """ - # 26 letters, 10 digits = 36 choices - # Remove O, 0, I, 1 => 32 choices - # 32 choices means we're just using the low 5 bit of each byte, - # so there's no bias introduced by using a modulo - chrs = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789" - - random_bytes = os.urandom(length) - return "".join([chrs[ord(random_bytes[i]) % 32] for i in xrange(length)]) + r = random.SystemRandom() + return "".join([r.choice(symbols) for _i in xrange(length)]) def last_octet(address): -- cgit From 1d69d499124317aa1a9cf7d4bc54db2ff0bc3be9 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 15 Mar 2011 14:33:45 -0400 Subject: refactor onset_files quota checking --- nova/compute/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/compute/api.py b/nova/compute/api.py index b6ef889f6..c11059a28 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -86,6 +86,8 @@ class API(base.Base): Raises a QuotaError if any limit is exceeded """ + if onset_files is None: + return limit = quota.allowed_onset_files(context) if len(onset_files) > limit: raise quota.QuotaError(code="OnsetFileLimitExceeded") @@ -96,7 +98,6 @@ class API(base.Base): raise quota.QuotaError(code="OnsetFilePathLimitExceeded") if len(content) > content_limit: raise quota.QuotaError(code="OnsetFileContentLimitExceeded") - return onset_files def create(self, context, instance_type, image_id, kernel_id=None, ramdisk_id=None, @@ -142,8 +143,7 @@ class API(base.Base): LOG.warn(msg) raise quota.QuotaError(msg, "MetadataLimitExceeded") - if onset_files is not None: - onset_files = self._check_onset_file_quota(context, onset_files) + self._check_onset_file_quota(context, onset_files) image = self.image_service.show(context, image_id) -- cgit From 0eaf02efd5fef3f77fced9c1a71c32a6f14f293f Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Tue, 15 Mar 2011 16:21:22 -0500 Subject: Add logging to lock check --- nova/utils.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'nova') diff --git a/nova/utils.py b/nova/utils.py index 87e726394..d6f9ba829 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -518,6 +518,9 @@ def synchronized(name): def wrap(f): @functools.wraps(f) def inner(*args, **kwargs): + LOG.debug(_("Attempting to grab %(lock)s for method " + "%(method)s..." % {"lock": name, + "method": f.__name__})) lock = lockfile.FileLock(os.path.join(FLAGS.lock_path, 'nova-%s.lock' % name)) with lock: -- cgit From e9ef6e04786a40d20f8022bec5d23d2e4503ce3a Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 15 Mar 2011 17:56:00 -0400 Subject: s/onset_files/injected_files/g --- nova/api/openstack/servers.py | 16 ++++----- nova/compute/api.py | 22 ++++++------ nova/compute/manager.py | 2 +- nova/db/sqlalchemy/models.py | 2 +- nova/quota.py | 30 ++++++++-------- nova/tests/api/openstack/test_servers.py | 48 ++++++++++++------------- nova/tests/test_quota.py | 60 ++++++++++++++++---------------- nova/virt/xenapi/vmops.py | 21 +++++------ 8 files changed, 101 insertions(+), 100 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index adb5c5f99..42fe13619 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -169,7 +169,7 @@ class Controller(wsgi.Controller): metadata.append({'key': k, 'value': v}) personality = env['server'].get('personality', []) - onset_files = self._get_onset_files(personality) + injected_files = self._get_injected_files(personality) try: instances = self.compute_api.create( @@ -183,7 +183,7 @@ class Controller(wsgi.Controller): key_name=key_pair['name'], key_data=key_pair['public_key'], metadata=metadata, - onset_files=onset_files) + injected_files=injected_files) except QuotaError as error: self._handle_quota_error(error) @@ -207,15 +207,15 @@ class Controller(wsgi.Controller): else: return self._deserialize(request.body, request.get_content_type()) - def _get_onset_files(self, personality): + def _get_injected_files(self, personality): """ - Create a list of onset files from the personality attribute + Create a list of injected files from the personality attribute - At this time, onset_files must be formatted as a list of + At this time, injected_files must be formatted as a list of (file_path, file_content) pairs for compatibility with the underlying compute service. """ - onset_files = [] + injected_files = [] for item in personality: try: path = item['path'] @@ -230,8 +230,8 @@ class Controller(wsgi.Controller): except TypeError: msg = 'Personality content for %s cannot be decoded' % path raise exc.HTTPBadRequest(explanation=msg) - onset_files.append((path, contents)) - return onset_files + injected_files.append((path, contents)) + return injected_files def _handle_quota_errors(self, error): """ diff --git a/nova/compute/api.py b/nova/compute/api.py index c11059a28..32577af82 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -80,20 +80,20 @@ class API(base.Base): topic, {"method": "get_network_topic", "args": {'fake': 1}}) - def _check_onset_file_quota(self, context, onset_files): + def _check_injected_file_quota(self, context, injected_files): """ - Enforce quota limits on onset files + Enforce quota limits on injected files Raises a QuotaError if any limit is exceeded """ - if onset_files is None: + if injected_files is None: return - limit = quota.allowed_onset_files(context) - if len(onset_files) > limit: + limit = quota.allowed_injected_files(context) + if len(injected_files) > limit: raise quota.QuotaError(code="OnsetFileLimitExceeded") - path_limit = quota.allowed_onset_file_path_bytes(context) - content_limit = quota.allowed_onset_file_content_bytes(context) - for path, content in onset_files: + path_limit = quota.allowed_injected_file_path_bytes(context) + content_limit = quota.allowed_injected_file_content_bytes(context) + for path, content in injected_files: if len(path) > path_limit: raise quota.QuotaError(code="OnsetFilePathLimitExceeded") if len(content) > content_limit: @@ -105,7 +105,7 @@ class API(base.Base): display_name='', display_description='', key_name=None, key_data=None, security_group='default', availability_zone=None, user_data=None, metadata=[], - onset_files=None): + injected_files=None): """Create the number of instances requested if quota and other arguments check out ok.""" @@ -143,7 +143,7 @@ class API(base.Base): LOG.warn(msg) raise quota.QuotaError(msg, "MetadataLimitExceeded") - self._check_onset_file_quota(context, onset_files) + self._check_injected_file_quota(context, injected_files) image = self.image_service.show(context, image_id) @@ -246,7 +246,7 @@ class API(base.Base): "args": {"topic": FLAGS.compute_topic, "instance_id": instance_id, "availability_zone": availability_zone, - "onset_files": onset_files}}) + "injected_files": injected_files}}) for group_id in security_groups: self.trigger_security_group_members_refresh(elevated, group_id) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 6bb169fa5..92deca813 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -179,7 +179,7 @@ class ComputeManager(manager.Manager): """Launch a new instance with specified options.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) - instance_ref.onset_files = kwargs.get('onset_files', []) + instance_ref.injected_files = kwargs.get('injected_files', []) if instance_ref['name'] in self.driver.list_instances(): raise exception.Error(_("Instance has already been created")) LOG.audit(_("instance %s: starting..."), instance_id, diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 162f6fded..1845e85eb 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -161,7 +161,7 @@ class Certificate(BASE, NovaBase): class Instance(BASE, NovaBase): """Represents a guest vm.""" __tablename__ = 'instances' - onset_files = [] + injected_files = [] id = Column(Integer, primary_key=True, autoincrement=True) diff --git a/nova/quota.py b/nova/quota.py index e0fb97542..2b24c0b5b 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -37,12 +37,12 @@ flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') flags.DEFINE_integer('quota_metadata_items', 128, 'number of metadata items allowed per instance') -flags.DEFINE_integer('quota_max_onset_files', 5, - 'number of onset files allowed') -flags.DEFINE_integer('quota_max_onset_file_content_bytes', 10 * 1024, - 'number of bytes allowed per onset file') -flags.DEFINE_integer('quota_max_onset_file_path_bytes', 255, - 'number of bytes allowed per onset file path') +flags.DEFINE_integer('quota_max_injected_files', 5, + 'number of injected files allowed') +flags.DEFINE_integer('quota_max_injected_file_content_bytes', 10 * 1024, + 'number of bytes allowed per injected file') +flags.DEFINE_integer('quota_max_injected_file_path_bytes', 255, + 'number of bytes allowed per injected file path') def get_quota(context, project_id): @@ -113,19 +113,19 @@ def allowed_metadata_items(context, num_metadata_items): return min(num_metadata_items, num_allowed_metadata_items) -def allowed_onset_files(context): - """Return the number of onset files allowed""" - return int(FLAGS.quota_max_onset_files) +def allowed_injected_files(context): + """Return the number of injected files allowed""" + return FLAGS.quota_max_injected_files -def allowed_onset_file_content_bytes(context): - """Return the number of bytes allowed per onset file content""" - return int(FLAGS.quota_max_onset_file_content_bytes) +def allowed_injected_file_content_bytes(context): + """Return the number of bytes allowed per injected file content""" + return FLAGS.quota_max_injected_file_content_bytes -def allowed_onset_file_path_bytes(context): - """Return the number of bytes allowed in an onset file path""" - return int(FLAGS.quota_max_onset_file_path_bytes) +def allowed_injected_file_path_bytes(context): + """Return the number of bytes allowed in an injected file path""" + return FLAGS.quota_max_injected_file_path_bytes class QuotaError(exception.ApiError): diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 7027c7ea3..253b84be9 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -834,13 +834,13 @@ class TestServerInstanceCreation(test.TestCase): class MockComputeAPI(object): def __init__(self): - self.onset_files = None + self.injected_files = None def create(self, *args, **kwargs): - if 'onset_files' in kwargs: - self.onset_files = kwargs['onset_files'] + if 'injected_files' in kwargs: + self.injected_files = kwargs['injected_files'] else: - self.onset_files = None + self.injected_files = None return [{'id': '1234', 'display_name': 'fakeinstance'}] def set_admin_password(self, *args, **kwargs): @@ -920,46 +920,46 @@ class TestServerInstanceCreation(test.TestCase): request = self._get_create_request_json(body_dict) compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) - return request, response, compute_api.onset_files + return request, response, compute_api.injected_files def _create_instance_with_personality_xml(self, personality): body_dict = self._create_personality_request_dict(personality) request = self._get_create_request_xml(body_dict) compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) - return request, response, compute_api.onset_files + return request, response, compute_api.injected_files def test_create_instance_with_no_personality(self): - request, response, onset_files = \ + request, response, injected_files = \ self._create_instance_with_personality_json(personality=None) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, []) + self.assertEquals(injected_files, []) def test_create_instance_with_no_personality_xml(self): - request, response, onset_files = \ + request, response, injected_files = \ self._create_instance_with_personality_xml(personality=None) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, []) + self.assertEquals(injected_files, []) def test_create_instance_with_personality(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Hello, World!"\n' b64contents = base64.b64encode(contents) personality = [(path, b64contents)] - request, response, onset_files = \ + request, response, injected_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [(path, contents)]) + self.assertEquals(injected_files, [(path, contents)]) def test_create_instance_with_personality_xml(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Hello, World!"\n' b64contents = base64.b64encode(contents) personality = [(path, b64contents)] - request, response, onset_files = \ + request, response, injected_files = \ self._create_instance_with_personality_xml(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [(path, contents)]) + self.assertEquals(injected_files, [(path, contents)]) def test_create_instance_with_personality_no_path(self): personality = [('/remove/this/path', @@ -970,7 +970,7 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.onset_files, None) + self.assertEquals(compute_api.injected_files, None) def _test_create_instance_with_personality_no_path_xml(self): personality = [('/remove/this/path', @@ -981,7 +981,7 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.onset_files, None) + self.assertEquals(compute_api.injected_files, None) def test_create_instance_with_personality_no_contents(self): personality = [('/test/path', @@ -992,7 +992,7 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.onset_files, None) + self.assertEquals(compute_api.injected_files, None) def test_create_instance_with_personality_not_a_list(self): personality = [('/test/path', base64.b64encode('test\ncontents\n'))] @@ -1003,16 +1003,16 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.onset_files, None) + self.assertEquals(compute_api.injected_files, None) def test_create_instance_with_personality_with_non_b64_content(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Oh no!"\n' personality = [(path, contents)] - request, response, onset_files = \ + request, response, injected_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 400) - self.assertEquals(onset_files, None) + self.assertEquals(injected_files, None) def test_create_instance_with_three_personalities(self): files = [ @@ -1023,19 +1023,19 @@ class TestServerInstanceCreation(test.TestCase): personality = [] for path, content in files: personality.append((path, base64.b64encode(content))) - request, response, onset_files = \ + request, response, injected_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, files) + self.assertEquals(injected_files, files) def test_create_instance_personality_empty_content(self): path = '/my/file/path' contents = '' personality = [(path, contents)] - request, response, onset_files = \ + request, response, injected_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [(path, contents)]) + self.assertEquals(injected_files, [(path, contents)]) def test_create_instance_admin_pass_json(self): request, response, dummy = \ diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index d94381aa2..c65bc459d 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -200,66 +200,66 @@ class QuotaTestCase(test.TestCase): image_id='fake', metadata=metadata) - def test_allowed_onset_files(self): + def test_allowed_injected_files(self): self.assertEqual( - quota.allowed_onset_files(self.context), - FLAGS.quota_max_onset_files) + quota.allowed_injected_files(self.context), + FLAGS.quota_max_injected_files) - def _create_with_onset_files(self, files): + def _create_with_injected_files(self, files): api = compute.API(image_service=self.StubImageService()) api.create(self.context, min_count=1, max_count=1, instance_type='m1.small', image_id='fake', - onset_files=files) + injected_files=files) - def test_no_onset_files(self): + def test_no_injected_files(self): api = compute.API(image_service=self.StubImageService()) api.create(self.context, instance_type='m1.small', image_id='fake') - def test_max_onset_files(self): + def test_max_injected_files(self): files = [] - for i in xrange(FLAGS.quota_max_onset_files): + for i in xrange(FLAGS.quota_max_injected_files): files.append(('/my/path%d' % i, 'config = test\n')) - self._create_with_onset_files(files) # no QuotaError + self._create_with_injected_files(files) # no QuotaError - def test_too_many_onset_files(self): + def test_too_many_injected_files(self): files = [] - for i in xrange(FLAGS.quota_max_onset_files + 1): + for i in xrange(FLAGS.quota_max_injected_files + 1): files.append(('/my/path%d' % i, 'my\ncontent%d\n' % i)) self.assertRaises(quota.QuotaError, - self._create_with_onset_files, files) + self._create_with_injected_files, files) - def test_allowed_onset_file_content_bytes(self): + def test_allowed_injected_file_content_bytes(self): self.assertEqual( - quota.allowed_onset_file_content_bytes(self.context), - FLAGS.quota_max_onset_file_content_bytes) + quota.allowed_injected_file_content_bytes(self.context), + FLAGS.quota_max_injected_file_content_bytes) - def test_max_onset_file_content_bytes(self): - max = FLAGS.quota_max_onset_file_content_bytes + def test_max_injected_file_content_bytes(self): + max = FLAGS.quota_max_injected_file_content_bytes content = ''.join(['a' for i in xrange(max)]) files = [('/test/path', content)] - self._create_with_onset_files(files) # no QuotaError + self._create_with_injected_files(files) # no QuotaError - def test_too_many_onset_file_content_bytes(self): - max = FLAGS.quota_max_onset_file_content_bytes + def test_too_many_injected_file_content_bytes(self): + max = FLAGS.quota_max_injected_file_content_bytes content = ''.join(['a' for i in xrange(max + 1)]) files = [('/test/path', content)] self.assertRaises(quota.QuotaError, - self._create_with_onset_files, files) + self._create_with_injected_files, files) - def test_allowed_onset_file_path_bytes(self): + def test_allowed_injected_file_path_bytes(self): self.assertEqual( - quota.allowed_onset_file_path_bytes(self.context), - FLAGS.quota_max_onset_file_path_bytes) + quota.allowed_injected_file_path_bytes(self.context), + FLAGS.quota_max_injected_file_path_bytes) - def test_max_onset_file_path_bytes(self): - max = FLAGS.quota_max_onset_file_path_bytes + def test_max_injected_file_path_bytes(self): + max = FLAGS.quota_max_injected_file_path_bytes path = ''.join(['a' for i in xrange(max)]) files = [(path, 'config = quotatest')] - self._create_with_onset_files(files) # no QuotaError + self._create_with_injected_files(files) # no QuotaError - def test_too_many_onset_file_path_bytes(self): - max = FLAGS.quota_max_onset_file_path_bytes + def test_too_many_injected_file_path_bytes(self): + max = FLAGS.quota_max_injected_file_path_bytes path = ''.join(['a' for i in xrange(max + 1)]) files = [(path, 'config = quotatest')] self.assertRaises(quota.QuotaError, - self._create_with_onset_files, files) + self._create_with_injected_files, files) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index d3fc335fe..488a61e8e 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -137,19 +137,20 @@ class VMOps(object): LOG.info(_('Spawning VM %(instance_name)s created %(vm_ref)s.') % locals()) - def _inject_onset_files(): - onset_files = instance.onset_files - if onset_files: + def _inject_files(): + injected_files = instance.injected_files + if injected_files: # Check if this is a JSON-encoded string and convert if needed. - if isinstance(onset_files, basestring): + if isinstance(injected_files, basestring): try: - onset_files = json.loads(onset_files) + injected_files = json.loads(injected_files) except ValueError: - LOG.exception(_("Invalid value for onset_files: '%s'") - % onset_files) - onset_files = [] + LOG.exception( + _("Invalid value for injected_files: '%s'") + % injected_files) + injected_files = [] # Inject any files, if specified - for path, contents in instance.onset_files: + for path, contents in instance.injected_files: LOG.debug(_("Injecting file path: '%s'") % path) self.inject_file(instance, path, contents) # NOTE(armando): Do we really need to do this in virt? @@ -165,7 +166,7 @@ class VMOps(object): if state == power_state.RUNNING: LOG.debug(_('Instance %s: booted'), instance_name) timer.stop() - _inject_onset_files() + _inject_files() return True except Exception, exc: LOG.warn(exc) -- cgit From 3cc78174e023b3f848b9c4b30468d356ee575ea6 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 15 Mar 2011 18:11:54 -0400 Subject: internationalization --- nova/api/openstack/servers.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 42fe13619..f618c31a0 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -221,15 +221,16 @@ class Controller(wsgi.Controller): path = item['path'] contents = item['contents'] except KeyError as key: - expl = 'Bad personality format: missing %s' % key + expl = _('Bad personality format: missing %s') % key raise exc.HTTPBadRequest(explanation=expl) except TypeError: - raise exc.HTTPBadRequest(explanation='Bad personality format') + expl = _('Bad personality format') + raise exc.HTTPBadRequest(explanation=expl) try: contents = base64.b64decode(contents) except TypeError: - msg = 'Personality content for %s cannot be decoded' % path - raise exc.HTTPBadRequest(explanation=msg) + expl = _('Personality content for %s cannot be decoded') % path + raise exc.HTTPBadRequest(explanation=expl) injected_files.append((path, contents)) return injected_files @@ -238,13 +239,13 @@ class Controller(wsgi.Controller): Reraise quota errors as api-specific http exceptions """ if error.code == "OnsetFileLimitExceeded": - expl = "Personality file limit exceeded" + expl = _("Personality file limit exceeded") raise exc.HTTPBadRequest(explanation=expl) if error.code == "OnsetFilePathLimitExceeded": - expl = "Personality file path too long" + expl = _("Personality file path too long") raise exc.HTTPBadRequest(explanation=expl) if error.code == "OnsetFileContentLimitExceeded": - expl = "Personality file content too long" + expl = _("Personality file content too long") raise exc.HTTPBadRequest(explanation=expl) # if the original error is okay, just reraise it raise error -- cgit From 70769dbe239c979d97154b88a33cb34d377d1196 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 15 Mar 2011 18:12:46 -0400 Subject: pep8 --- nova/tests/api/openstack/test_servers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 253b84be9..a92c0f590 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -806,7 +806,7 @@ ZSBwYXR0ZXJucyBiZWhpbmQgYWxsIGNsb3VkcywgYW5kIHlv\ dSB3aWxsIGtub3csIHRvbywgd2hlbiB5b3UgbGlmdCB5b3Vy\ c2VsZiBoaWdoIGVub3VnaCB0byBzZWUgYmV5b25kIGhvcml6\ b25zLiINCg0KLVJpY2hhcmQgQmFjaA==""", - } + }, ], }} request = self.deserializer.deserialize(serial_request) -- cgit From e237b4a5653384688b16f7fd2c0708eaec4b9ec7 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 15 Mar 2011 19:11:21 -0400 Subject: ignore differently-named nodes in personality and metadata parsing --- nova/api/openstack/servers.py | 10 +- nova/tests/api/openstack/test_servers.py | 164 ++++++++++++++++++++++--------- 2 files changed, 126 insertions(+), 48 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index f618c31a0..ea88f1fdc 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -575,7 +575,7 @@ class ServerCreateRequestXMLDeserializer(object): if metadata_node is None: return None metadata = {} - for meta_node in metadata_node.childNodes: + for meta_node in self._find_children_named(metadata_node, "meta"): key = meta_node.getAttribute("key") metadata[key] = self._extract_text(meta_node) return metadata @@ -587,7 +587,7 @@ class ServerCreateRequestXMLDeserializer(object): if personality_node is None: return None personality = [] - for file_node in personality_node.childNodes: + for file_node in self._find_children_named(personality_node, "file"): item = {} if file_node.hasAttribute("path"): item["path"] = file_node.getAttribute("path") @@ -602,6 +602,12 @@ class ServerCreateRequestXMLDeserializer(object): return node return None + def _find_children_named(self, parent, name): + """Return all of a nodes children who have the given name""" + for node in parent.childNodes: + if node.nodeName == name: + yield node + def _extract_text(self, node): """Get the text field contained by the given node""" if len(node.childNodes) == 1: diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index a92c0f590..ed37cb705 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -610,7 +610,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_minimal_request(self): serial_request = """ -""" request = self.deserializer.deserialize(serial_request) expected = {"server": { @@ -622,8 +622,10 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_empty_metadata(self): serial_request = """ -""" + + +""" request = self.deserializer.deserialize(serial_request) expected = {"server": { "name": "new-server-test", @@ -635,8 +637,10 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_empty_personality(self): serial_request = """ -""" + + +""" request = self.deserializer.deserialize(serial_request) expected = {"server": { "name": "new-server-test", @@ -648,9 +652,11 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_empty_metadata_and_personality(self): serial_request = """ -\ -""" + + + +""" request = self.deserializer.deserialize(serial_request) expected = {"server": { "name": "new-server-test", @@ -663,9 +669,11 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_empty_metadata_and_personality_reversed(self): serial_request = """ -\ -""" + + + +""" request = self.deserializer.deserialize(serial_request) expected = {"server": { "name": "new-server-test", @@ -678,28 +686,47 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_one_personality(self): serial_request = """ -\ -aabbccdd""" + + + aabbccdd + +""" request = self.deserializer.deserialize(serial_request) expected = [{"path": "/etc/conf", "contents": "aabbccdd"}] self.assertEquals(request["server"]["personality"], expected) def test_request_with_two_personalities(self): serial_request = """ -\ -aabbccdd\ + +aabbccdd abcd""" request = self.deserializer.deserialize(serial_request) expected = [{"path": "/etc/conf", "contents": "aabbccdd"}, {"path": "/etc/sudoers", "contents": "abcd"}] self.assertEquals(request["server"]["personality"], expected) + def test_request_second_personality_node_ignored(self): + serial_request = """ + + + aabbccdd + + + anything + +""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": "aabbccdd"}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_one_personality_missing_path(self): serial_request = """ -\ + aabbccdd""" request = self.deserializer.deserialize(serial_request) expected = [{"contents": "aabbccdd"}] @@ -707,8 +734,8 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_one_personality_empty_contents(self): serial_request = """ -\ + """ request = self.deserializer.deserialize(serial_request) expected = [{"path": "/etc/conf", "contents": ""}] @@ -716,8 +743,8 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_one_personality_empty_contents_variation(self): serial_request = """ -\ + """ request = self.deserializer.deserialize(serial_request) expected = [{"path": "/etc/conf", "contents": ""}] @@ -725,57 +752,101 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_one_metadata(self): serial_request = """ -\ -beta""" + + + beta + +""" request = self.deserializer.deserialize(serial_request) expected = {"alpha": "beta"} self.assertEquals(request["server"]["metadata"], expected) def test_request_with_two_metadata(self): serial_request = """ -\ -betabar\ -""" + + + beta + bar + +""" request = self.deserializer.deserialize(serial_request) expected = {"alpha": "beta", "foo": "bar"} self.assertEquals(request["server"]["metadata"], expected) def test_request_with_metadata_missing_value(self): serial_request = """ -\ -""" + + + + +""" request = self.deserializer.deserialize(serial_request) expected = {"alpha": ""} self.assertEquals(request["server"]["metadata"], expected) + def test_request_with_two_metadata_missing_value(self): + serial_request = """ + + + + + +""" + request = self.deserializer.deserialize(serial_request) + expected = {"alpha": "", "delta": ""} + self.assertEquals(request["server"]["metadata"], expected) + def test_request_with_metadata_missing_key(self): serial_request = """ -\ -beta""" + + + beta + +""" request = self.deserializer.deserialize(serial_request) expected = {"": "beta"} self.assertEquals(request["server"]["metadata"], expected) + def test_request_with_two_metadata_missing_key(self): + serial_request = """ + + + beta + gamma + +""" + request = self.deserializer.deserialize(serial_request) + expected = {"":"gamma"} + self.assertEquals(request["server"]["metadata"], expected) + def test_request_with_metadata_duplicate_key(self): serial_request = """ -\ -barbaz\ -""" + + + bar + baz + +""" request = self.deserializer.deserialize(serial_request) expected = {"foo": "baz"} self.assertEquals(request["server"]["metadata"], expected) def test_canonical_request_from_docs(self): serial_request = """ -\ -Apache1\ -\ + + + Apache1 + + + \ ICAgICAgDQoiQSBjbG91ZCBkb2VzIG5vdCBrbm93IHdoeSBp\ dCBtb3ZlcyBpbiBqdXN0IHN1Y2ggYSBkaXJlY3Rpb24gYW5k\ IGF0IHN1Y2ggYSBzcGVlZC4uLkl0IGZlZWxzIGFuIGltcHVs\ @@ -784,8 +855,9 @@ QnV0IHRoZSBza3kga25vd3MgdGhlIHJlYXNvbnMgYW5kIHRo\ ZSBwYXR0ZXJucyBiZWhpbmQgYWxsIGNsb3VkcywgYW5kIHlv\ dSB3aWxsIGtub3csIHRvbywgd2hlbiB5b3UgbGlmdCB5b3Vy\ c2VsZiBoaWdoIGVub3VnaCB0byBzZWUgYmV5b25kIGhvcml6\ -b25zLiINCg0KLVJpY2hhcmQgQmFjaA==\ -""" +b25zLiINCg0KLVJpY2hhcmQgQmFjaA== + +""" expected = {"server": { "name": "new-server-test", "imageId": "1", -- cgit From fc07caece79e379b6d6f2a3220806af9271e349b Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 15 Mar 2011 19:23:46 -0400 Subject: pep8 --- nova/tests/api/openstack/test_servers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index ed37cb705..9a6f2c052 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -722,7 +722,6 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): expected = [{"path": "/etc/conf", "contents": "aabbccdd"}] self.assertEquals(request["server"]["personality"], expected) - def test_request_with_one_personality_missing_path(self): serial_request = """ """ request = self.deserializer.deserialize(serial_request) - expected = {"":"gamma"} + expected = {"": "gamma"} self.assertEquals(request["server"]["metadata"], expected) def test_request_with_metadata_duplicate_key(self): -- cgit From 016669543a1f6d4ffc281637ba98c6b6fe30be82 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Wed, 16 Mar 2011 10:38:48 +0100 Subject: Fix unknown exception error in euca-get-ajax-console --- nova/virt/libvirt_conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 7994e9547..d7bdc3faa 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -502,7 +502,7 @@ class LibvirtConnection(object): cmd = 'netcat', '0.0.0.0', port, '-w', '1' try: stdout, stderr = utils.execute(*cmd, process_input='') - except ProcessExecutionError: + except exception.ProcessExecutionError: return port raise Exception(_('Unable to find an open port')) -- cgit From ba35831c1f66c424e9495642ba23e9d2742a339e Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Wed, 16 Mar 2011 10:58:02 +0100 Subject: added correct path to cpu information (tested on a system with 1 installed cpu package) --- nova/virt/libvirt_conn.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 7994e9547..9943b742a 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -984,18 +984,18 @@ class LibvirtConnection(object): xml = self._conn.getCapabilities() xml = libxml2.parseDoc(xml) - nodes = xml.xpathEval('//cpu') + nodes = xml.xpathEval('//host/cpu') if len(nodes) != 1: raise exception.Invalid(_("Invalid xml. '' must be 1," "but %d\n") % len(nodes) + xml.serialize()) cpu_info = dict() - cpu_info['arch'] = xml.xpathEval('//cpu/arch')[0].getContent() - cpu_info['model'] = xml.xpathEval('//cpu/model')[0].getContent() - cpu_info['vendor'] = xml.xpathEval('//cpu/vendor')[0].getContent() + cpu_info['arch'] = xml.xpathEval('//host/cpu/arch')[0].getContent() + cpu_info['model'] = xml.xpathEval('//host/cpu/model')[0].getContent() + cpu_info['vendor'] = xml.xpathEval('//host/cpu/vendor')[0].getContent() - topology_node = xml.xpathEval('//cpu/topology')[0].get_properties() + topology_node = xml.xpathEval('//host/cpu/topology')[0].get_properties() topology = dict() while topology_node != None: name = topology_node.get_name() @@ -1009,7 +1009,7 @@ class LibvirtConnection(object): raise exception.Invalid(_("Invalid xml: topology(%(topology)s) " "must have %(ks)s") % locals()) - feature_nodes = xml.xpathEval('//cpu/feature') + feature_nodes = xml.xpathEval('//host/cpu/feature') features = list() for nodes in feature_nodes: features.append(nodes.get_properties().getContent()) -- cgit From 7fa96f6292ff7d63621fe024b1ef45b1a1996121 Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Wed, 16 Mar 2011 14:49:18 -0400 Subject: Re-commit r804 --- nova/tests/api/openstack/test_servers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 40026a615..cde2fc036 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -216,7 +216,7 @@ class ServersTest(test.TestCase): servers = json.loads(res.body)['servers'] self.assertEqual([s['id'] for s in servers], [1, 2]) - def _test_create_instance_helper(self, with_key_pair): + def _test_create_instance_helper(self): def instance_create(context, inst): return {'id': '1', 'display_name': 'server_test'} @@ -272,11 +272,11 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) def test_create_instance(self): - self._test_create_instance_helper(True) + self._test_create_instance_helper() def test_create_instance_no_key_pair(self): - fakes.stub_out_key_pair_funcs(self.stubs, False) - self._test_create_instance_helper(False) + fakes.stub_out_key_pair_funcs(self.stubs, have_key_pair=False) + self._test_create_instance_helper() def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') -- cgit From 663c1726d9a96540b8fd729223fcb34d7cf3cdf7 Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Wed, 16 Mar 2011 14:49:25 -0400 Subject: Re-commit r805 --- nova/tests/api/openstack/test_servers.py | 1 + 1 file changed, 1 insertion(+) (limited to 'nova') diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index cde2fc036..ad36fa551 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -217,6 +217,7 @@ class ServersTest(test.TestCase): self.assertEqual([s['id'] for s in servers], [1, 2]) def _test_create_instance_helper(self): + """Shared implementation for tests below that create instance""" def instance_create(context, inst): return {'id': '1', 'display_name': 'server_test'} -- cgit From 157ea09c03148ff4615bae27ca3f276a05620825 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Wed, 16 Mar 2011 19:54:15 +0100 Subject: fixed pep8 issue --- nova/virt/libvirt_conn.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 9943b742a..4e17555f4 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -995,7 +995,8 @@ class LibvirtConnection(object): cpu_info['model'] = xml.xpathEval('//host/cpu/model')[0].getContent() cpu_info['vendor'] = xml.xpathEval('//host/cpu/vendor')[0].getContent() - topology_node = xml.xpathEval('//host/cpu/topology')[0].get_properties() + topology_node = xml.xpathEval('//host/cpu/topology')[0]\ + .get_properties() topology = dict() while topology_node != None: name = topology_node.get_name() -- cgit From abc6c82449dfc46a33dcd8190840e51f44b5b930 Mon Sep 17 00:00:00 2001 From: Sandy Walsh Date: Thu, 17 Mar 2011 07:30:22 -0700 Subject: Replaced capability flags with List --- nova/api/openstack/zones.py | 4 ++-- nova/flags.py | 5 +++-- nova/tests/api/openstack/test_zones.py | 6 +++--- 3 files changed, 8 insertions(+), 7 deletions(-) (limited to 'nova') diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index 547920901..ebfc7743c 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -71,9 +71,9 @@ class Controller(wsgi.Controller): items = api.API.get_zone_capabilities(req.environ['nova.context']) zone = dict(name=FLAGS.zone_name) - caps = FLAGS.zone_capabilities.split(';') + caps = FLAGS.zone_capabilities for cap in caps: - key_values = cap.split(':') + key_values = cap.split('=') zone[key_values[0]] = key_values[1] for item, (min_value, max_value) in items.iteritems(): zone[item] = "%s,%s" % (min_value, max_value) diff --git a/nova/flags.py b/nova/flags.py index c05cef373..3a8ec1a39 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -358,5 +358,6 @@ DEFINE_string('node_availability_zone', 'nova', 'availability zone of this node') DEFINE_string('zone_name', 'nova', 'name of this zone') -DEFINE_string('zone_capabilities', 'hypervisor:xenserver;os:linux', - 'Key/Value tags which represent capabilities of this zone') +DEFINE_list('zone_capabilities', + ['hypervisor=xenserver;kvm', 'os=linux;windows'], + 'Key/Multi-value list representng capabilities of this zone') diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 5e3aee4a7..12d39fd29 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -178,7 +178,7 @@ class ZonesTest(test.TestCase): def test_zone_info(self): FLAGS.zone_name = 'darksecret' - FLAGS.zone_capabilities = 'cap1:a,b;cap2:c,d' + FLAGS.zone_capabilities = ['cap1=a;b', 'cap2=c;d'] self.stubs.Set(api, '_call_scheduler', zone_caps) body = dict(zone=dict(username='zeb', password='sneaky')) @@ -188,5 +188,5 @@ class ZonesTest(test.TestCase): res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) self.assertEqual(res_dict['zone']['name'], 'darksecret') - self.assertEqual(res_dict['zone']['cap1'], 'a,b') - self.assertEqual(res_dict['zone']['cap2'], 'c,d') + self.assertEqual(res_dict['zone']['cap1'], 'a;b') + self.assertEqual(res_dict['zone']['cap2'], 'c;d') -- cgit