From 7c8096384507908a5e583f4554d0fc765ae5f2eb Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Thu, 27 Jan 2011 20:39:33 +0900 Subject: adding testcode --- bin/nova-api | 2 + nova/compute/manager.py | 66 ++-- nova/db/sqlalchemy/api.py | 49 +-- nova/db/sqlalchemy/models.py | 12 +- nova/scheduler/driver.py | 53 ++-- nova/scheduler/manager.py | 28 +- nova/tests/test_compute.py | 305 ++++++++++++++++++ nova/tests/test_scheduler.py | 722 +++++++++++++++++++++++++++++++++++++++++++ nova/tests/test_service.py | 61 +++- nova/tests/test_virt.py | 520 ++++++++++++++++++++++++++++++- nova/virt/fake.py | 12 +- nova/virt/libvirt_conn.py | 159 +++++----- nova/virt/xenapi_conn.py | 14 +- nova/volume/manager.py | 2 +- 14 files changed, 1809 insertions(+), 196 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 7b4fbeab1..fba09889f 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -37,11 +37,13 @@ gettext.install('nova', unicode=1) from nova import flags from nova import log as logging from nova import wsgi +from nova import utils logging.basicConfig() LOG = logging.getLogger('nova.api') LOG.setLevel(logging.DEBUG) +utils.default_flagfile() FLAGS = flags.FLAGS API_ENDPOINTS = ['ec2', 'osapi'] diff --git a/nova/compute/manager.py b/nova/compute/manager.py index efb5753aa..4acba7153 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -125,12 +125,12 @@ class ComputeManager(manager.Manager): """Insert compute node specific information to DB.""" try: - service_ref = db.service_get_by_args(ctxt, - host, - binary) + service_ref = self.db.service_get_by_args(ctxt, + host, + binary) except exception.NotFound: - msg = _(("""Cannot insert compute manager specific info""" - """Because no service record found.""")) + msg = _(("""Cannot insert compute manager specific info,""" + """ Because no service record found.""")) raise exception.Invalid(msg) # Updating host information @@ -141,14 +141,14 @@ class ComputeManager(manager.Manager): version = self.driver.get_hypervisor_version() cpu_info = self.driver.get_cpu_info() - db.service_update(ctxt, - service_ref['id'], - {'vcpus': vcpu, - 'memory_mb': memory_mb, - 'local_gb': local_gb, - 'hypervisor_type': hypervisor, - 'hypervisor_version': version, - 'cpu_info': cpu_info}) + self.db.service_update(ctxt, + service_ref['id'], + {'vcpus': vcpu, + 'memory_mb': memory_mb, + 'local_gb': local_gb, + 'hypervisor_type': hypervisor, + 'hypervisor_version': version, + 'cpu_info': cpu_info}) def _update_state(self, context, instance_id): """Update the state of an instance from the driver info.""" @@ -596,22 +596,22 @@ class ComputeManager(manager.Manager): """ Check the host cpu is compatible to a cpu given by xml.""" return self.driver.compare_cpu(cpu_info) - def pre_live_migration(self, context, instance_id, dest): + def pre_live_migration(self, context, instance_id): """Any preparation for live migration at dst host.""" # Getting instance info - instance_ref = db.instance_get(context, instance_id) + instance_ref = self.db.instance_get(context, instance_id) ec2_id = instance_ref['hostname'] # Getting fixed ips - fixed_ip = db.instance_get_fixed_address(context, instance_id) + fixed_ip = self.db.instance_get_fixed_address(context, instance_id) if not fixed_ip: msg = _('%s(%s) doesnt have fixed_ip') % (instance_id, ec2_id) raise exception.NotFound(msg) # If any volume is mounted, prepare here. if len(instance_ref['volumes']) == 0: - logging.info(_("%s has no volume.") % ec2_id) + LOG.info(_("%s has no volume."), ec2_id) else: for v in instance_ref['volumes']: self.volume_manager.setup_compute_volume(context, v['id']) @@ -634,7 +634,7 @@ class ComputeManager(manager.Manager): """executes live migration.""" # Get instance for error handling. - instance_ref = db.instance_get(context, instance_id) + instance_ref = self.db.instance_get(context, instance_id) ec2_id = instance_ref['hostname'] try: @@ -647,27 +647,27 @@ class ComputeManager(manager.Manager): "args": {'instance_id': instance_id}}) # Asking dest host to preparing live migration. - compute_topic = db.queue_get_for(context, - FLAGS.compute_topic, - dest) + compute_topic = self.db.queue_get_for(context, + FLAGS.compute_topic, + dest) rpc.call(context, - compute_topic, - {"method": "pre_live_migration", - "args": {'instance_id': instance_id, - 'dest': dest}}) + compute_topic, + {"method": "pre_live_migration", + "args": {'instance_id': instance_id}}) except Exception, e: + print e msg = _('Pre live migration for %s failed at %s') - logging.error(msg, ec2_id, dest) - db.instance_set_state(context, - instance_id, - power_state.RUNNING, - 'running') + LOG.error(msg, ec2_id, dest) + self.db.instance_set_state(context, + instance_id, + power_state.RUNNING, + 'running') for v in instance_ref['volumes']: - db.volume_update(context, - v['id'], - {'status': 'in-use'}) + self.db.volume_update(context, + v['id'], + {'status': 'in-use'}) # e should be raised. just calling "raise" may raise NotFound. raise e diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 248a46f65..1cdd5a286 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -897,41 +897,42 @@ def instance_get_all_by_host(context, hostname): @require_context -def _instance_get_sum_by_host_and_project(context, column, hostname, proj_id): +def instance_get_vcpu_sum_by_host_and_project(context, hostname, proj_id): session = get_session() - result = session.query(models.Instance).\ - filter_by(host=hostname).\ - filter_by(project_id=proj_id).\ - filter_by(deleted=can_read_deleted(context)).\ - value(column) - if not result: + filter_by(host=hostname).\ + filter_by(project_id=proj_id).\ + filter_by(deleted=False).\ + value(func.sum(models.Instance.vcpus)) + if None == result: return 0 return result -@require_context -def instance_get_vcpu_sum_by_host_and_project(context, hostname, proj_id): - return _instance_get_sum_by_host_and_project(context, - 'vcpus', - hostname, - proj_id) - - @require_context def instance_get_memory_sum_by_host_and_project(context, hostname, proj_id): - return _instance_get_sum_by_host_and_project(context, - 'memory_mb', - hostname, - proj_id) - + session = get_session() + result = session.query(models.Instance).\ + filter_by(host=hostname).\ + filter_by(project_id=proj_id).\ + filter_by(deleted=False).\ + value(func.sum(models.Instance.memory_mb)) + if None == result: + return 0 + return result @require_context def instance_get_disk_sum_by_host_and_project(context, hostname, proj_id): - return _instance_get_sum_by_host_and_project(context, - 'local_gb', - hostname, - proj_id) + session = get_session() + result = session.query(models.Instance).\ + filter_by(host=hostname).\ + filter_by(project_id=proj_id).\ + filter_by(deleted=False).\ + value(func.sum(models.Instance.local_gb)) + if None == result: + return 0 + return result + @require_context diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index b28c64b59..7c40d5596 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -161,11 +161,11 @@ class Service(BASE, NovaBase): # The below items are compute node only. # -1 or None is inserted for other service. - vcpus = Column(Integer, nullable=False, default=-1) - memory_mb = Column(Integer, nullable=False, default=-1) - local_gb = Column(Integer, nullable=False, default=-1) - hypervisor_type = Column(String(128)) - hypervisor_version = Column(Integer, nullable=False, default=-1) + vcpus = Column(Integer, nullable=True) + memory_mb = Column(Integer, nullable=True) + local_gb = Column(Integer, nullable=True) + hypervisor_type = Column(String(128), nullable=True) + hypervisor_version = Column(Integer, nullable=True) # Note(masumotok): Expected Strings example: # # '{"arch":"x86_64", "model":"Nehalem", @@ -174,7 +174,7 @@ class Service(BASE, NovaBase): # # Points are "json translatable" and it must have all # dictionary keys above. - cpu_info = Column(String(512)) + cpu_info = Column(Text(), nullable=True) class Certificate(BASE, NovaBase): diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 65745093b..d4ad42388 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -69,11 +69,10 @@ class Scheduler(object): raise NotImplementedError(_("Must implement a fallback schedule")) def schedule_live_migration(self, context, instance_id, dest): - """ live migration method """ + """live migration method""" # Whether instance exists and running instance_ref = db.instance_get(context, instance_id) - ec2_id = instance_ref['hostname'] # Checking instance. self._live_migration_src_check(context, instance_ref) @@ -159,48 +158,45 @@ class Scheduler(object): def _live_migration_common_check(self, context, instance_ref, dest): """ - Live migration check routine. - Below pre-checkings are followed by - http://wiki.libvirt.org/page/TodoPreMigrationChecks + Live migration check routine. + Below pre-checkings are followed by + http://wiki.libvirt.org/page/TodoPreMigrationChecks """ # Checking dest exists. dservice_refs = db.service_get_all_by_host(context, dest) if len(dservice_refs) <= 0: - msg = _('%s does not exists.') - raise exception.Invalid(msg % dest) + raise exception.Invalid(_('%s does not exists.') % dest) dservice_ref = dservice_refs[0] # Checking original host( where instance was launched at) exists. - orighost = instance_ref['launched_on'] - oservice_refs = db.service_get_all_by_host(context, orighost) + oservice_refs = db.service_get_all_by_host(context, + instance_ref['launched_on']) if len(oservice_refs) <= 0: msg = _('%s(where instance was launched at) does not exists.') - raise exception.Invalid(msg % orighost) + raise exception.Invalid(msg % instance_ref['launched_on']) oservice_ref = oservice_refs[0] # Checking hypervisor is same. - otype = oservice_ref['hypervisor_type'] - dtype = dservice_ref['hypervisor_type'] - if otype != dtype: + if oservice_ref['hypervisor_type'] != dservice_ref['hypervisor_type']: msg = _('Different hypervisor type(%s->%s)') - raise exception.Invalid(msg % (otype, dtype)) + raise exception.Invalid(msg % (oservice_ref['hypervisor_type'], + dservice_ref['hypervisor_type'])) # Checkng hypervisor version. - oversion = oservice_ref['hypervisor_version'] - dversion = dservice_ref['hypervisor_version'] - if oversion > dversion: + if oservice_ref['hypervisor_version'] > \ + dservice_ref['hypervisor_version']: msg = _('Older hypervisor version(%s->%s)') - raise exception.Invalid(msg % (oversion, dversion)) + raise exception.Invalid(msg % (oservice_ref['hypervisor_version'], + dservice_ref['hypervisor_version'])) # Checking cpuinfo. - cpu_info = oservice_ref['cpu_info'] try: rpc.call(context, db.queue_get_for(context, FLAGS.compute_topic, dest), {"method": 'compare_cpu', - "args": {'cpu_info': cpu_info}}) + "args": {'cpu_info': oservice_ref['cpu_info']}}) except rpc.RemoteError, e: msg = _(("""%s doesnt have compatibility to %s""" @@ -211,7 +207,7 @@ class Scheduler(object): raise e def has_enough_resource(self, context, instance_ref, dest): - """ Check if destination host has enough resource for live migration""" + """Check if destination host has enough resource for live migration""" # Getting instance information ec2_id = instance_ref['hostname'] @@ -222,28 +218,27 @@ class Scheduler(object): # Gettin host information service_refs = db.service_get_all_by_host(context, dest) if len(service_refs) <= 0: - msg = _('%s does not exists.') - raise exception.Invalid(msg % dest) + raise exception.Invalid(_('%s does not exists.') % dest) service_ref = service_refs[0] total_cpu = int(service_ref['vcpus']) total_mem = int(service_ref['memory_mb']) total_hdd = int(service_ref['local_gb']) - instances_ref = db.instance_get_all_by_host(context, dest) - for i_ref in instances_ref: + instances_refs = db.instance_get_all_by_host(context, dest) + for i_ref in instances_refs: total_cpu -= int(i_ref['vcpus']) total_mem -= int(i_ref['memory_mb']) total_hdd -= int(i_ref['local_gb']) # Checking host has enough information - logging.debug('host(%s) remains vcpu:%s mem:%s hdd:%s,' % + logging.debug(_('host(%s) remains vcpu:%s mem:%s hdd:%s,') % (dest, total_cpu, total_mem, total_hdd)) - logging.debug('instance(%s) has vcpu:%s mem:%s hdd:%s,' % + logging.debug(_('instance(%s) has vcpu:%s mem:%s hdd:%s,') % (ec2_id, vcpus, mem, hdd)) if total_cpu <= vcpus or total_mem <= mem or total_hdd <= hdd: - msg = '%s doesnt have enough resource for %s' % (dest, ec2_id) - raise exception.NotEmpty(msg) + raise exception.NotEmpty(_('%s is not capable to migrate %s') % + (dest, ec2_id)) logging.debug(_('%s has_enough_resource() for %s') % (dest, ec2_id)) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 1cc767a03..a181225a6 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -73,17 +73,13 @@ class SchedulerManager(manager.Manager): # Based on bear design summit discussion, # just put this here for bexar release. def show_host_resource(self, context, host, *args): - """ show the physical/usage resource given by hosts.""" + """show the physical/usage resource given by hosts.""" - services = db.service_get_all_by_host(context, host) - if len(services) == 0: - return {'ret': False, 'msg': 'No such Host'} - - compute = [s for s in services if s['topic'] == 'compute'] - if 0 == len(compute): - service_ref = services[0] - else: - service_ref = compute[0] + computes = db.service_get_all_compute_sorted(context) + computes = [s for s,v in computes if s['host'] == host] + if 0 == len(computes): + return {'ret': False, 'msg': 'No such Host or not compute node.'} + service_ref = computes[0] # Getting physical resource information h_resource = {'vcpus': service_ref['vcpus'], @@ -92,13 +88,15 @@ class SchedulerManager(manager.Manager): # Getting usage resource information u_resource = {} - instances_ref = db.instance_get_all_by_host(context, - service_ref['host']) + instances_refs = db.instance_get_all_by_host(context, + service_ref['host']) - if 0 == len(instances_ref): - return {'ret': True, 'phy_resource': h_resource, 'usage': {}} + if 0 == len(instances_refs): + return {'ret': True, + 'phy_resource': h_resource, + 'usage': u_resource} - project_ids = [i['project_id'] for i in instances_ref] + project_ids = [i['project_id'] for i in instances_refs] project_ids = list(set(project_ids)) for p_id in project_ids: vcpus = db.instance_get_vcpu_sum_by_host_and_project(context, diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 09f6ee94a..344c2d2b5 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -20,6 +20,7 @@ Tests For Compute """ import datetime +import mox from nova import compute from nova import context @@ -27,9 +28,12 @@ from nova import db from nova import exception from nova import flags from nova import log as logging +from nova import rpc from nova import test from nova import utils from nova.auth import manager +from nova.compute import manager as compute_manager +from nova.compute import power_state LOG = logging.getLogger('nova.tests.compute') @@ -219,3 +223,304 @@ class ComputeTestCase(test.TestCase): self.assertEqual(ret_val, None) self.compute.terminate_instance(self.context, instance_id) + + def test_update_service_exception(self): + """nova-compute updates Serivce table on DB like below. + nova.service.Serivce.start -> + nova.compute.ComputeManager.update_service. + This testcase confirms if no record found on Service + table, exception can be raised. + """ + host = 'foo' + binary = 'nova-compute' + dbmock = self.mox.CreateMock(db) + dbmock.service_get_by_args(mox.IgnoreArg(), + mox.StrContains(host), + mox.StrContains(binary)).\ + AndRaise(exception.NotFound()) + self.compute.db = dbmock + self.mox.ReplayAll() + try: + self.compute.update_service('dummy', host, binary) + except exception.Invalid, e: + msg = 'Cannot insert compute manager specific info' + c1 = ( 0 <= e.message.find(msg)) + self.assertTrue(c1) + self.mox.ResetAll() + + def test_update_service_success(self): + """nova-compute updates Serivce table on DB like below. + nova.service.Serivce.start -> + nova.compute.ComputeManager.update_service. + In this method, vcpus/memory_mb/local_gb/hypervisor_type/ + hypervisor_version/cpu_info should be changed. + Based on this specification, this testcase confirms + if this method finishes successfully, + meaning self.db.service_update is called with dictinary + + {'vcpu':aaa, 'memory_mb':bbb, 'local_gb':ccc, + 'hypervisor_type':ddd, 'hypervisor_version':eee, + 'cpu_info':fff} + + Since each value of above dict can be obtained through + driver(different depends on environment), + only dictionary keys are checked. + """ + + def dic_key_check(dic): + validkey = ['vcpus', 'memory_mb', 'local_gb', + 'hypervisor_type', 'hypervisor_version', 'cpu_info'] + return (list(set(validkey)) == list(set(dic.keys()))) + + host = 'foo' + binary = 'nova-compute' + service_ref = {'id':1, 'binary':'nova-compute', 'topic':'compute'} + dbmock = self.mox.CreateMock(db) + dbmock.service_get_by_args(mox.IgnoreArg(), + mox.StrContains(host), + mox.StrContains(binary)).\ + AndReturn(service_ref) + dbmock.service_update(mox.IgnoreArg(), + service_ref['id'], + mox.Func(dic_key_check)) + + self.compute.db = dbmock + self.mox.ReplayAll() + try: + self.compute.update_service('dummy', host, binary) + except exception.Invalid, e: + msg = 'Cannot insert compute manager specific info' + c1 = ( 0 <= e.message.find(msg)) + self.assertTrue(c1) + self.mox.ResetAll() + + def _setup_other_managers(self): + self.volume_manager = utils.import_object(FLAGS.volume_manager) + self.network_manager = utils.import_object(FLAGS.network_manager) + self.compute_driver = utils.import_object(FLAGS.compute_driver) + + def test_pre_live_migration_instance_has_no_fixed_ip(self): + """ + if instances that are intended to be migrated doesnt have fixed_ip + (not happens usually), pre_live_migration has to raise Exception. + """ + instance_ref={'id':1, 'volumes':[{'id':1}, {'id':2}], + 'hostname':'i-000000001'} + c = context.get_admin_context() + i_id = instance_ref['id'] + + dbmock = self.mox.CreateMock(db) + dbmock.instance_get(c, i_id).AndReturn(instance_ref) + dbmock.instance_get_fixed_address(c, i_id).AndReturn(None) + + self.compute.db = dbmock + self.mox.ReplayAll() + self.assertRaises(exception.NotFound, + self.compute.pre_live_migration, + c, instance_ref['id']) + self.mox.ResetAll() + + def test_pre_live_migration_instance_has_volume(self): + """if any volumes are attached to the instances that are + intended to be migrated, setup_compute_volume must be + called because aoe module should be inserted at destination + host. This testcase checks on it. + """ + instance_ref={'id':1, 'volumes':[{'id':1}, {'id':2}], + 'hostname':'i-000000001'} + c = context.get_admin_context() + i_id=instance_ref['id'] + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + volmock = self.mox.CreateMock(self.volume_manager) + netmock = self.mox.CreateMock(self.network_manager) + drivermock = self.mox.CreateMock(self.compute_driver) + + dbmock.instance_get(c, i_id).AndReturn(instance_ref) + dbmock.instance_get_fixed_address(c, i_id).AndReturn('dummy') + for i in range(len(instance_ref['volumes'])): + vid = instance_ref['volumes'][i]['id'] + volmock.setup_compute_volume(c, vid).InAnyOrder('g1') + netmock.setup_compute_network(c, instance_ref['id']) + drivermock.ensure_filtering_rules_for_instance(instance_ref) + + self.compute.db = dbmock + self.compute.volume_manager = volmock + self.compute.network_manager = netmock + self.compute.driver = drivermock + + self.mox.ReplayAll() + ret = self.compute.pre_live_migration(c, i_id) + self.assertEqual(ret, None) + self.mox.ResetAll() + + def test_pre_live_migration_instance_has_no_volume(self): + """if any volumes are not attached to the instances that are + intended to be migrated, log message should be appears + because administrator can proove instance conditions before + live_migration if any trouble occurs. + """ + instance_ref={'id':1, 'volumes':[], 'hostname':'i-20000001'} + c = context.get_admin_context() + i_id = instance_ref['id'] + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + netmock = self.mox.CreateMock(self.network_manager) + drivermock = self.mox.CreateMock(self.compute_driver) + + dbmock.instance_get(c, i_id).AndReturn(instance_ref) + dbmock.instance_get_fixed_address(c, i_id).AndReturn('dummy') + self.mox.StubOutWithMock(compute_manager.LOG, 'info') + compute_manager.LOG.info(_("%s has no volume."), instance_ref['hostname']) + netmock.setup_compute_network(c, i_id) + drivermock.ensure_filtering_rules_for_instance(instance_ref) + + self.compute.db = dbmock + self.compute.network_manager = netmock + self.compute.driver = drivermock + + self.mox.ReplayAll() + ret = self.compute.pre_live_migration(c, i_id) + self.assertEqual(ret, None) + self.mox.ResetAll() + + def test_live_migration_instance_has_volume(self): + """Any volumes are mounted by instances to be migrated are found, + vblade health must be checked before starting live-migration. + And that is checked by check_for_export(). + This testcase confirms check_for_export() is called. + """ + instance_ref={'id':1, 'volumes':[{'id':1}, {'id':2}], 'hostname':'i-00000001'} + c = context.get_admin_context() + dest='dummydest' + i_id = instance_ref['id'] + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + drivermock = self.mox.CreateMock(self.compute_driver) + + dbmock.instance_get(c, instance_ref['id']).AndReturn(instance_ref) + self.mox.StubOutWithMock(rpc, 'call') + rpc.call(c, FLAGS.volume_topic, + {"method": "check_for_export", + "args": {'instance_id': i_id}}).InAnyOrder('g1') + rpc.call(c, db.queue_get_for(c, FLAGS.compute_topic, dest), + {"method": "pre_live_migration", + "args": {'instance_id': i_id}}).InAnyOrder('g1') + + self.compute.db = dbmock + self.compute.driver = drivermock + self.mox.ReplayAll() + ret = self.compute.live_migration(c, i_id, dest) + self.assertEqual(ret, None) + self.mox.ResetAll() + + def test_live_migration_instance_has_volume_and_exception(self): + """In addition to test_live_migration_instance_has_volume testcase, + this testcase confirms if any exception raises from check_for_export(). + Then, valid seaquence of this method should recovering instance/volumes + status(ex. instance['state_description'] is changed from 'migrating' + -> 'running', was changed by scheduler) + """ + instance_ref={'id':1, 'volumes':[{'id':1}, {'id':2}], + 'hostname':'i-000000001'} + dest='dummydest' + c = context.get_admin_context() + i_id = instance_ref['id'] + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + drivermock = self.mox.CreateMock(self.compute_driver) + + dbmock.instance_get(c, instance_ref['id']).AndReturn(instance_ref) + self.mox.StubOutWithMock(rpc, 'call') + rpc.call(c, FLAGS.volume_topic, + {"method": "check_for_export", + "args": {'instance_id': i_id}}).InAnyOrder('g1') + compute_topic = db.queue_get_for(c, FLAGS.compute_topic, dest) + dbmock.queue_get_for(c, FLAGS.compute_topic, dest).AndReturn(compute_topic) + rpc.call(c, db.queue_get_for(c, FLAGS.compute_topic, dest), + {"method": "pre_live_migration", + "args": {'instance_id': i_id}}).\ + InAnyOrder('g1').AndRaise(rpc.RemoteError('du', 'mm', 'y')) + self.mox.StubOutWithMock(compute_manager.LOG, 'error') + compute_manager.LOG.error('Pre live migration for %s failed at %s', + instance_ref['hostname'], dest) + dbmock.instance_set_state(c, i_id, power_state.RUNNING, 'running') + for i in range(len(instance_ref['volumes'])): + vid = instance_ref['volumes'][i]['id'] + dbmock.volume_update(c, vid, {'status': 'in-use'}) + + self.compute.db = dbmock + self.compute.driver = drivermock + self.mox.ReplayAll() + self.assertRaises(rpc.RemoteError, + self.compute.live_migration, + c, i_id, dest) + self.mox.ResetAll() + + def test_live_migration_instance_has_no_volume_and_exception(self): + """Simpler than test_live_migration_instance_has_volume_and_exception""" + + instance_ref={'id':1, 'volumes':[], 'hostname':'i-000000001'} + dest='dummydest' + c = context.get_admin_context() + i_id = instance_ref['id'] + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + drivermock = self.mox.CreateMock(self.compute_driver) + + dbmock.instance_get(c, instance_ref['id']).AndReturn(instance_ref) + self.mox.StubOutWithMock(rpc, 'call') + compute_topic = db.queue_get_for(c, FLAGS.compute_topic, dest) + dbmock.queue_get_for(c, FLAGS.compute_topic, dest).AndReturn(compute_topic) + rpc.call(c, compute_topic, + {"method": "pre_live_migration", + "args": {'instance_id': i_id}}).\ + AndRaise(rpc.RemoteError('du', 'mm', 'y')) + self.mox.StubOutWithMock(compute_manager.LOG, 'error') + compute_manager.LOG.error('Pre live migration for %s failed at %s', + instance_ref['hostname'], dest) + dbmock.instance_set_state(c, i_id, power_state.RUNNING, 'running') + + self.compute.db = dbmock + self.compute.driver = drivermock + self.mox.ReplayAll() + self.assertRaises(rpc.RemoteError, + self.compute.live_migration, + c, i_id, dest) + self.mox.ResetAll() + + def test_live_migration_instance_has_volume(self): + """Simpler version than test_live_migration_instance_has_volume.""" + instance_ref={'id':1, 'volumes':[{'id':1}, {'id':2}], + 'hostname':'i-000000001'} + c = context.get_admin_context() + dest='dummydest' + i_id = instance_ref['id'] + + self._setup_other_managers() + dbmock = self.mox.CreateMock(db) + drivermock = self.mox.CreateMock(self.compute_driver) + + dbmock.instance_get(c, i_id).AndReturn(instance_ref) + self.mox.StubOutWithMock(rpc, 'call') + rpc.call(c, FLAGS.volume_topic, + {"method": "check_for_export", + "args": {'instance_id': i_id}}).InAnyOrder('g1') + compute_topic = db.queue_get_for(c, FLAGS.compute_topic, dest) + dbmock.queue_get_for(c, FLAGS.compute_topic, dest).AndReturn(compute_topic) + rpc.call(c, compute_topic, + {"method": "pre_live_migration", + "args": {'instance_id': i_id}}).InAnyOrder('g1') + drivermock.live_migration(c, instance_ref, dest) + + self.compute.db = dbmock + self.compute.driver = drivermock + self.mox.ReplayAll() + ret = self.compute.live_migration(c, i_id, dest) + self.assertEqual(ret, None) + self.mox.ResetAll() diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 9d458244b..c62bca9b1 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -20,10 +20,12 @@ Tests For Scheduler """ import datetime +import mox from mox import IgnoreArg from nova import context from nova import db +from nova import exception from nova import flags from nova import service from nova import test @@ -32,6 +34,8 @@ from nova import utils from nova.auth import manager as auth_manager from nova.scheduler import manager from nova.scheduler import driver +from nova.compute import power_state +from nova.db.sqlalchemy import models FLAGS = flags.FLAGS @@ -75,7 +79,102 @@ class SchedulerTestCase(test.TestCase): 'args': {'num': 7}}) self.mox.ReplayAll() scheduler.named_method(ctxt, 'topic', num=7) + + def test_show_host_resource_host_not_exit(self): + """ + A testcase of driver.has_enough_resource + given host does not exists. + """ + scheduler = manager.SchedulerManager() + dest = 'dummydest' + ctxt = context.get_admin_context() + + self.mox.StubOutWithMock(manager, 'db', use_mock_anything=True) + manager.db.service_get_all_compute_sorted(mox.IgnoreArg()).\ + AndReturn([]) + + self.mox.ReplayAll() + result = scheduler.show_host_resource(ctxt, dest) + # ret should be dict + keys = ['ret', 'msg'] + c1 = list(set(result.keys())) == list(set(keys)) + c2 = not result['ret'] + c3 = result['msg'].find('No such Host or not compute node') <= 0 + self.assertTrue( c1 and c2 and c3) + self.mox.UnsetStubs() + + def test_show_host_resource_no_project(self): + """ + A testcase of driver.show_host_resource + no instance stays on the given host + """ + scheduler = manager.SchedulerManager() + dest = 'dummydest' + ctxt = context.get_admin_context() + r0 = {'vcpus':16, 'memory_mb':32, 'local_gb':100} + service_ref = {'id':1, 'host':dest} + service_ref.update(r0) + + self.mox.StubOutWithMock(manager, 'db', use_mock_anything=True) + manager.db.service_get_all_compute_sorted(mox.IgnoreArg()).\ + AndReturn([(service_ref, 0)]) + manager.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([]) + + self.mox.ReplayAll() + result = scheduler.show_host_resource(ctxt, dest) + # ret should be dict + keys = ['ret', 'phy_resource', 'usage'] + c1 = list(set(result.keys())) == list(set(keys)) + c2 = result['ret'] + c3 = result['phy_resource'] == r0 + c4 = result['usage'] == {} + self.assertTrue( c1 and c2 and c3 and c4) + self.mox.UnsetStubs() + + def test_show_host_resource_works_correctly(self): + """ + A testcase of driver.show_host_resource + to make sure everything finished with no error. + """ + scheduler = manager.SchedulerManager() + dest = 'dummydest' + ctxt = context.get_admin_context() + r0 = {'vcpus':16, 'memory_mb':32, 'local_gb':100} + r1 = {'vcpus':10, 'memory_mb':4, 'local_gb':20} + r2 = {'vcpus':10, 'memory_mb':20, 'local_gb':30} + service_ref = {'id':1, 'host':dest} + service_ref.update(r0) + instance_ref2 = {'id':2, 'project_id':'p-01', 'host':'dummy'} + instance_ref2.update(r1) + instance_ref3 = {'id':3, 'project_id':'p-02', 'host':'dummy'} + instance_ref3.update(r1) + self.mox.StubOutWithMock(manager, 'db', use_mock_anything=True) + manager.db.service_get_all_compute_sorted(mox.IgnoreArg()).\ + AndReturn([(service_ref, 0)]) + manager.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([instance_ref2, instance_ref3]) + for p in ['p-01', 'p-02']: + manager.db.instance_get_vcpu_sum_by_host_and_project( + ctxt, dest, p).AndReturn(r2['vcpus']) + manager.db.instance_get_memory_sum_by_host_and_project( + ctxt, dest, p).AndReturn(r2['memory_mb']) + manager.db.instance_get_disk_sum_by_host_and_project( + ctxt, dest, p).AndReturn(r2['local_gb']) + + self.mox.ReplayAll() + result = scheduler.show_host_resource(ctxt, dest) + # ret should be dict + keys = ['ret', 'phy_resource', 'usage'] + c1 = list(set(result.keys())) == list(set(keys)) + c2 = result['ret'] + c3 = result['phy_resource'] == r0 + c4 = result['usage'].keys() == ['p-01', 'p-02'] + c5 = result['usage']['p-01'] == r2 + c6 = result['usage']['p-02'] == r2 + self.assertTrue( c1 and c2 and c3 and c4 and c5 and c6) + self.mox.UnsetStubs() class ZoneSchedulerTestCase(test.TestCase): """Test case for zone scheduler""" @@ -384,3 +483,626 @@ class SimpleDriverTestCase(test.TestCase): volume2.delete_volume(self.context, volume_id) volume1.kill() volume2.kill() + + def test_scheduler_live_migraiton_with_volume(self): + """ + driver.scheduler_live_migration finishes successfully + (volumes are attached to instances) + This testcase make sure schedule_live_migration + changes instance state from 'running' -> 'migrating' + """ + driver_i = self.scheduler.driver + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-00000001', 'host':'dummy', + 'volumes':[{'id':1}, {'id':2}]} + dest = 'dummydest' + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + # must be IgnoreArg() because scheduler changes ctxt's memory address + driver.db.instance_get(mox.IgnoreArg(), i_ref['id']).AndReturn(i_ref) + + self.mox.StubOutWithMock(driver_i, '_live_migration_src_check') + driver_i._live_migration_src_check(mox.IgnoreArg(), i_ref) + self.mox.StubOutWithMock(driver_i, '_live_migration_dest_check') + driver_i._live_migration_dest_check(mox.IgnoreArg(), i_ref, dest) + self.mox.StubOutWithMock(driver_i, '_live_migration_common_check') + driver_i._live_migration_common_check(mox.IgnoreArg(), i_ref, dest) + driver.db.instance_set_state(mox.IgnoreArg(), i_ref['id'], + power_state.PAUSED, 'migrating') + for v in i_ref['volumes']: + driver.db.volume_update(mox.IgnoreArg(), v['id'], + {'status': 'migrating'}) + self.mox.StubOutWithMock(rpc, 'cast', use_mock_anything=True) + kwargs={'instance_id':i_ref['id'], 'dest':dest} + rpc.cast(ctxt, db.queue_get_for(ctxt, topic, i_ref['host']), + {"method": 'live_migration', + "args": kwargs}) + + self.mox.ReplayAll() + self.scheduler.live_migration(ctxt, topic, + instance_id=i_ref['id'], dest=dest) + self.mox.UnsetStubs() + + def test_scheduler_live_migraiton_no_volume(self): + """ + driver.scheduler_live_migration finishes successfully + (volumes are attached to instances) + This testcase make sure schedule_live_migration + changes instance state from 'running' -> 'migrating' + """ + driver_i = self.scheduler.driver + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', 'volumes':[]} + dest = 'dummydest' + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + # must be IgnoreArg() because scheduler changes ctxt's memory address + driver.db.instance_get(mox.IgnoreArg(), i_ref['id']).AndReturn(i_ref) + self.mox.StubOutWithMock(driver_i, '_live_migration_src_check') + driver_i._live_migration_src_check(mox.IgnoreArg(), i_ref) + self.mox.StubOutWithMock(driver_i, '_live_migration_dest_check') + driver_i._live_migration_dest_check(mox.IgnoreArg(), i_ref, dest) + self.mox.StubOutWithMock(driver_i, '_live_migration_common_check') + driver_i._live_migration_common_check(mox.IgnoreArg(), i_ref, dest) + driver.db.instance_set_state(mox.IgnoreArg(), i_ref['id'], + power_state.PAUSED, 'migrating') + self.mox.StubOutWithMock(rpc, 'cast', use_mock_anything=True) + kwargs={'instance_id':i_ref['id'], 'dest':dest} + rpc.cast(ctxt, db.queue_get_for(ctxt, topic, i_ref['host']), + {"method": 'live_migration', + "args": kwargs}) + + self.mox.ReplayAll() + self.scheduler.live_migration(ctxt, topic, + instance_id=i_ref['id'], dest=dest) + self.mox.UnsetStubs() + + def test_live_migraiton_src_check_instance_not_running(self): + """ + A testcase of driver._live_migration_src_check. + The instance given by instance_id is not running. + """ + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + dest = 'dummydest' + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', + 'volumes':[], 'state_description':'migrating', + 'state':power_state.RUNNING} + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_src_check(ctxt, i_ref) + except exception.Invalid, e: + self.assertTrue(e.message.find('is not running') > 0) + self.mox.UnsetStubs() + + def test_live_migraiton_src_check_volume_node_not_alive(self): + """ + A testcase of driver._live_migration_src_check. + Volume node is not alive if any volumes are attached to + the given instance. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', + 'volumes':[{'id':1}, {'id':2}], + 'state_description':'running', 'state':power_state.RUNNING} + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_topic(mox.IgnoreArg(), 'volume').\ + AndReturn([]) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_src_check(ctxt, i_ref) + except exception.Invalid, e: + self.assertTrue(e.message.find('volume node is not alive') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_src_check_volume_node_not_alive(self): + """ + A testcase of driver._live_migration_src_check. + The testcase make sure src-compute node is alive. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', 'volumes':[], + 'state_description':'running', 'state':power_state.RUNNING} + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_topic(mox.IgnoreArg(), 'compute').\ + AndReturn([]) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_src_check(ctxt, i_ref) + except exception.Invalid, e: + self.assertTrue(e.message.find('is not alive') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_src_check_works_correctly(self): + """ + A testcase of driver._live_migration_src_check. + The testcase make sure everything finished with no error. + """ + driver_i = self.scheduler.driver + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', 'volumes':[], + 'state_description':'running', 'state':power_state.RUNNING} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('host', i_ref['host']) + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_topic(mox.IgnoreArg(), 'compute').\ + AndReturn([service_ref]) + self.mox.StubOutWithMock(driver_i, 'service_is_up') + driver_i.service_is_up(service_ref).AndReturn(True) + + self.mox.ReplayAll() + ret = driver_i._live_migration_src_check(ctxt, i_ref) + self.assertTrue(ret == None) + self.mox.UnsetStubs() + + def test_live_migraiton_dest_check_service_not_exists(self): + """ + A testcase of driver._live_migration_dst_check. + Destination host does not exist. + """ + driver_i = self.scheduler.driver + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('host', i_ref['host']) + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([]) + + self.mox.ReplayAll() + try: + driver_i._live_migration_dest_check(ctxt, i_ref, dest) + except exception.Invalid, e: + self.assertTrue(e.message.find('does not exists') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_dest_check_service_isnot_compute(self): + """ + A testcase of driver._live_migration_dst_check. + Destination host does not provide compute. + """ + driver_i = self.scheduler.driver + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('host', i_ref['host']) + service_ref.__setitem__('topic', 'api') + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + + self.mox.ReplayAll() + try: + driver_i._live_migration_dest_check(ctxt, i_ref, dest) + except exception.Invalid, e: + self.assertTrue(e.message.find('must be compute node') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_dest_check_service_not_alive(self): + """ + A testcase of driver._live_migration_dst_check. + Destination host compute service is not alive. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('host', i_ref['host']) + service_ref.__setitem__('topic', 'compute') + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + self.mox.StubOutWithMock(self.scheduler.driver, 'service_is_up') + self.scheduler.driver.service_is_up(service_ref).AndReturn(False) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_dest_check(ctxt, i_ref, dest) + except exception.Invalid, e: + self.assertTrue(e.message.find('is not alive') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_dest_check_service_same_host(self): + """ + A testcase of driver._live_migration_dst_check. + Destination host is same as src host. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummydest'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('host', i_ref['host']) + service_ref.__setitem__('topic', 'compute') + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + self.mox.StubOutWithMock(self.scheduler.driver, 'service_is_up') + self.scheduler.driver.service_is_up(service_ref).AndReturn(True) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_dest_check(ctxt, i_ref, dest) + except exception.Invalid, e: + self.assertTrue(e.message.find('is running now. choose other host') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_dest_check_service_works_correctly(self): + """ + A testcase of driver._live_migration_dst_check. + The testcase make sure everything finished with no error. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummydest'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('host', i_ref['host']) + service_ref.__setitem__('topic', 'compute') + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + self.mox.StubOutWithMock(self.scheduler.driver, 'service_is_up') + self.scheduler.driver.service_is_up(service_ref).AndReturn(True) + self.mox.StubOutWithMock(self.scheduler.driver, 'has_enough_resource') + self.scheduler.driver.has_enough_resource(mox.IgnoreArg(), i_ref, dest) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_dest_check(ctxt, i_ref, dest) + except exception.Invalid, e: + self.assertTrue(e.message.find('is running now. choose other host') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_common_check_service_dest_not_exists(self): + """ + A testcase of driver._live_migration_common_check. + Destination host does not exist. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([]) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_common_check(ctxt, + i_ref, + dest) + except exception.Invalid, e: + self.assertTrue(e.message.find('does not exists') >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_common_check_service_orig_not_exists(self): + """ + A testcase of driver._live_migration_common_check. + Original host(an instance launched on) does not exist. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', 'launched_on':'h1'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('topic', 'compute') + service_ref.__setitem__('host', i_ref['host']) + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.service_get_all_by_host(mox.IgnoreArg(), + i_ref['launched_on']).\ + AndReturn([]) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_common_check(ctxt, + i_ref, + dest) + except exception.Invalid, e: + msg = 'where instance was launched at) does not exists' + self.assertTrue(e.message.find(msg) >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_common_check_service_different_hypervisor(self): + """ + A testcase of driver._live_migration_common_check. + Original host and dest host has different hypervisor type. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', + 'host':'dummy', 'launched_on':'h1'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('topic', 'compute') + service_ref.__setitem__('hypervisor_type', 'kvm') + service_ref2 = models.Service() + service_ref2.__setitem__('id', 2) + service_ref2.__setitem__('hypervisor_type', 'xen') + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.service_get_all_by_host(mox.IgnoreArg(), + i_ref['launched_on']).\ + AndReturn([service_ref2]) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_common_check(ctxt, + i_ref, + dest) + except exception.Invalid, e: + msg = 'Different hypervisor type' + self.assertTrue(e.message.find(msg) >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_common_check_service_different_version(self): + """ + A testcase of driver._live_migration_common_check. + Original host and dest host has different hypervisor version. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', + 'host':'dummy', 'launched_on':'h1'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('topic', 'compute') + service_ref.__setitem__('hypervisor_version', 12000) + service_ref2 = models.Service() + service_ref2.__setitem__('id', 2) + service_ref2.__setitem__('hypervisor_version', 12001) + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.service_get_all_by_host(mox.IgnoreArg(), + i_ref['launched_on']).\ + AndReturn([service_ref2]) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_common_check(ctxt, + i_ref, + dest) + except exception.Invalid, e: + msg = 'Older hypervisor version' + self.assertTrue(e.message.find(msg) >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_common_check_service_checking_cpuinfo_fail(self): + """ + A testcase of driver._live_migration_common_check. + Original host and dest host has different hypervisor version. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', + 'host':'dummy', 'launched_on':'h1'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('topic', 'compute') + service_ref.__setitem__('hypervisor_version', 12000) + service_ref2 = models.Service() + service_ref2.__setitem__('id', 2) + service_ref2.__setitem__('hypervisor_version', 12000) + service_ref2.__setitem__('cpuinfo', 'info') + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.service_get_all_by_host(mox.IgnoreArg(), + i_ref['launched_on']).\ + AndReturn([service_ref2]) + driver.db.queue_get_for(mox.IgnoreArg(), FLAGS.compute_topic, dest) + self.mox.StubOutWithMock(driver, 'rpc', use_mock_anything=True) + driver.rpc.call(mox.IgnoreArg(), mox.IgnoreArg(), + {"method": 'compare_cpu', + "args": {'cpu_info': service_ref2['cpu_info']}}).\ + AndRaise(rpc.RemoteError('doesnt have compatibility to', '', '')) + + self.mox.ReplayAll() + try: + self.scheduler.driver._live_migration_common_check(ctxt, + i_ref, + dest) + except rpc.RemoteError, e: + msg = 'doesnt have compatibility to' + self.assertTrue(e.message.find(msg) >= 0) + self.mox.UnsetStubs() + + def test_live_migraiton_common_check_service_works_correctly(self): + """ + A testcase of driver._live_migration_common_check. + The testcase make sure everything finished with no error. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + i_ref = {'id':1, 'hostname':'i-01', + 'host':'dummy', 'launched_on':'h1'} + service_ref = models.Service() + service_ref.__setitem__('id', 1) + service_ref.__setitem__('topic', 'compute') + service_ref.__setitem__('hypervisor_version', 12000) + service_ref2 = models.Service() + service_ref2.__setitem__('id', 2) + service_ref2.__setitem__('hypervisor_version', 12000) + service_ref2.__setitem__('cpuinfo', 'info') + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.service_get_all_by_host(mox.IgnoreArg(), + i_ref['launched_on']).\ + AndReturn([service_ref2]) + driver.db.queue_get_for(mox.IgnoreArg(), FLAGS.compute_topic, dest) + self.mox.StubOutWithMock(driver, 'rpc', use_mock_anything=True) + driver.rpc.call(mox.IgnoreArg(), mox.IgnoreArg(), + {"method": 'compare_cpu', + "args": {'cpu_info': service_ref2['cpu_info']}}) + + self.mox.ReplayAll() + ret = self.scheduler.driver._live_migration_common_check(ctxt, + i_ref, + dest) + self.assertTrue(ret == None) + self.mox.UnsetStubs() + + def test_has_enough_resource_lack_resource_vcpu(self): + """ + A testcase of driver.has_enough_resource. + Lack of vcpu.(boundary check) + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} + i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', + 'vcpus':6, 'memory_mb':8, 'local_gb':10} + i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([i_ref2, i_ref3]) + + self.mox.ReplayAll() + try: + self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) + except exception.NotEmpty, e: + msg = 'is not capable to migrate' + self.assertTrue(e.message.find(msg) >= 0) + self.mox.UnsetStubs() + + def test_has_enough_resource_lack_resource_memory(self): + """ + A testcase of driver.has_enough_resource. + Lack of memory_mb.(boundary check) + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} + i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':16, 'local_gb':10} + i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([i_ref2, i_ref3]) + + self.mox.ReplayAll() + try: + self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) + except exception.NotEmpty, e: + msg = 'is not capable to migrate' + self.assertTrue(e.message.find(msg) >= 0) + self.mox.UnsetStubs() + self.mox.UnsetStubs() + + def test_has_enough_resource_lack_resource_disk(self): + """ + A testcase of driver.has_enough_resource. + Lack of local_gb.(boundary check) + """ + scheduler = manager.SchedulerManager() + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} + i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':80} + i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([i_ref2, i_ref3]) + + self.mox.ReplayAll() + try: + self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) + except exception.NotEmpty, e: + msg = 'is not capable to migrate' + self.assertTrue(e.message.find(msg) >= 0) + self.mox.UnsetStubs() + + def test_has_enough_resource_works_correctly(self): + """ + A testcase of driver.has_enough_resource + to make sure everything finished with no error. + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} + i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', + 'vcpus':5, 'memory_mb':8, 'local_gb':10} + + self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) + driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([service_ref]) + driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ + AndReturn([i_ref2, i_ref3]) + + self.mox.ReplayAll() + ret = self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) + self.assertTrue(ret == None) + self.mox.UnsetStubs() diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index a67c8d1e8..a147e69b4 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -30,6 +30,7 @@ from nova import rpc from nova import test from nova import service from nova import manager +from nova.compute import manager as compute_manager FLAGS = flags.FLAGS flags.DEFINE_string("fake_manager", "nova.tests.test_service.FakeManager", @@ -41,7 +42,20 @@ class FakeManager(manager.Manager): def test_method(self): return 'manager' - +# temporary variable to store host/binary/self.mox from each method to fake class. +global_host = None +global_binary = None +global_mox = None +class FakeComputeManager(compute_manager.ComputeManager): + """Fake computemanager manager for tests""" + + def __init__(self, compute_driver=None, *args, **kwargs): + global ghost, gbinary, gmox + self.update_service(mox.IgnoreArg(), mox.StrContains(ghost), mox.StrContains(gbinary)) + gmox.ReplayAll() + super(FakeComputeManager, self).__init__(compute_driver, *args, **kwargs) + + class ExtendedService(service.Service): def test_method(self): return 'service' @@ -258,3 +272,48 @@ class ServiceTestCase(test.TestCase): serv.report_state() self.assert_(not serv.model_disconnected) + + def test_compute_can_update_services(self): + """ + Test nova-compute successfully updated Service table on DB. + Doing so, self.manager.update_service must be called + if 'self.binary == nova-compute', and this testcase checks on it. + """ + host = 'foo' + binary = 'nova-compute' + topic = 'compute1' + service_create = {'host': host, + 'binary': binary, + 'topic': topic, + 'report_count': 0, + 'availability_zone': 'nova'} + service_ref = {'host': host, + 'binary': binary, + 'topic': topic, + 'report_count': 0, + 'availability_zone': 'nova', + 'id': 1} + + service.db.service_get_by_args(mox.IgnoreArg(), + host, + binary).AndRaise(exception.NotFound()) + service.db.service_create(mox.IgnoreArg(), + service_create).AndReturn(service_ref) + self.mox.StubOutWithMock(compute_manager.ComputeManager, 'update_service') + + + global ghost, gbinary, gmox + ghost = host + gbinary = binary + gmox = self.mox + + serv = service.Service(host, + binary, + topic, + 'nova.tests.test_service.FakeComputeManager') + # ReplayAll has been executed FakeComputeManager.__init__() + #self.mox.ReplayAll() + serv.start() + serv.stop() + + diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index afdc89ba2..177e8f021 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -14,21 +14,29 @@ # License for the specific language governing permissions and limitations # under the License. +import mox + from xml.etree.ElementTree import fromstring as xml_to_tree from xml.dom.minidom import parseString as xml_to_dom from nova import context from nova import db +from nova import exception from nova import flags from nova import test +from nova import logging from nova import utils from nova.api.ec2 import cloud from nova.auth import manager +from nova.db.sqlalchemy import models +from nova.compute import power_state from nova.virt import libvirt_conn FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') +libvirt = None +libxml2 = None class LibvirtConnTestCase(test.TestCase): def setUp(self): @@ -52,6 +60,38 @@ class LibvirtConnTestCase(test.TestCase): 'bridge': 'br101', 'instance_type': 'm1.small'} + def _driver_dependent_test_setup(self): + """ + Setup method. + Call this method at the top of each testcase method, + if the testcase is necessary libvirt and cheetah. + """ + try : + global libvirt + global libxml2 + libvirt_conn.libvirt = __import__('libvirt') + libvirt_conn.libxml2 = __import__('libxml2') + libvirt_conn._late_load_cheetah() + libvirt = __import__('libvirt') + except ImportError, e: + logging.warn("""This test has not been done since """ + """using driver-dependent library Cheetah/libvirt/libxml2.""") + raise e + + # inebitable mocks for calling + #nova.virt.libvirt_conn.LibvirtConnection.__init__ + nwmock = self.mox.CreateMock(libvirt_conn.NWFilterFirewall) + self.mox.StubOutWithMock(libvirt_conn, 'NWFilterFirewall', + use_mock_anything=True) + libvirt_conn.NWFilterFirewall(mox.IgnoreArg()).AndReturn(nwmock) + + obj = utils.import_object(FLAGS.firewall_driver) + fwmock = self.mox.CreateMock(obj) + self.mox.StubOutWithMock(libvirt_conn, 'utils', + use_mock_anything=True) + libvirt_conn.utils.import_object(FLAGS.firewall_driver).AndReturn(fwmock) + return nwmock, fwmock + def test_xml_and_uri_no_ramdisk_no_kernel(self): instance_data = dict(self.test_instance) self._check_xml_and_uri(instance_data, @@ -188,9 +228,8 @@ class LibvirtConnTestCase(test.TestCase): expected_result, '%s failed common check %d' % (xml, i)) - # This test is supposed to make sure we don't override a specifically - # set uri - # + # This test is supposed to make sure we don't override a specifically set uri + # # Deliberately not just assigning this string to FLAGS.libvirt_uri and # checking against that later on. This way we make sure the # implementation doesn't fiddle around with the FLAGS. @@ -202,6 +241,480 @@ class LibvirtConnTestCase(test.TestCase): uri = conn.get_uri() self.assertEquals(uri, testuri) + def test_get_memory_mb(self): + """ + Check if get_memory_mb returns memory value + Connection/OS/driver differenct does not matter for this method, + so everyone can execute for checking. + """ + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertTrue(0 < conn.get_memory_mb()) + self.mox.UnsetStubs() + + def test_get_cpu_info_works_correctly(self): + """ + Check if get_cpu_info works correctly. + (in case libvirt.getCapabilities() works correctly) + """ + xml=("""x86_64Nehalem""" + """Intel""" + """""" + """""" + """""" + """""" + """""" + """""" + """""") + + try: + self._driver_dependent_test_setup() + except: + return + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', use_mock_anything=True) + libvirt_conn.LibvirtConnection._conn.getCapabilities().AndReturn(xml) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertTrue(0 < len(conn.get_cpu_info())) + self.mox.UnsetStubs() + + def test_get_cpu_info_inappropreate_xml(self): + """ + Check if get_cpu_info raises exception + in case libvirt.getCapabilities() returns wrong xml + (in case of xml doesnt have tag) + """ + xml=("""x86_64Nehalem""" + """Intel""" + """""" + """""" + """""" + """""" + """""" + """""" + """""") + + try: + self._driver_dependent_test_setup() + except: + return + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', use_mock_anything=True) + libvirt_conn.LibvirtConnection._conn.getCapabilities().AndReturn(xml) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + try: + conn.get_cpu_info() + except exception.Invalid, e: + c1 = ( 0 <= e.message.find('Invalid xml') ) + self.assertTrue(c1) + self.mox.UnsetStubs() + + def test_get_cpu_info_inappropreate_xml2(self): + """ + Check if get_cpu_info raises exception + in case libvirt.getCapabilities() returns wrong xml + (in case of xml doesnt have inproper tag + meaning missing "socket" attribute) + """ + xml=("""x86_64Nehalem""" + """Intel""" + """""" + """""" + """""" + """""" + """""" + """""" + """""") + + try: + self._driver_dependent_test_setup() + except: + return + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', use_mock_anything=True) + libvirt_conn.LibvirtConnection._conn.getCapabilities().AndReturn(xml) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + try: + conn.get_cpu_info() + except exception.Invalid, e: + c1 = ( 0 <= e.message.find('Invalid xml: topology') ) + self.assertTrue(c1) + self.mox.UnsetStubs() + + def test_compare_cpu_works_correctly(self): + """Calling libvirt.compute_cpu() and works correctly """ + + t = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ + """"topology":{"cores":"%s", "threads":"%s", """ + """"sockets":"%s"}, "features":[%s]}""") + cpu_info = t % ('x86', 'model', 'vendor', '2', '1', '4', '"tm"') + + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', use_mock_anything=True) + libvirt_conn.LibvirtConnection._conn.compareCPU(mox.IgnoreArg(),0).AndReturn(1) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertTrue( None== conn.compare_cpu(cpu_info)) + self.mox.UnsetStubs() + + def test_compare_cpu_raises_exception(self): + """ + Libvirt-related exception occurs when calling + libvirt.compare_cpu(). + """ + t = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ + """"topology":{"cores":"%s", "threads":"%s", """ + """"sockets":"%s"}, "features":[%s]}""") + cpu_info = t % ('x86', 'model', 'vendor', '2', '1', '4', '"tm"') + + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', + use_mock_anything=True) + libvirt_conn.LibvirtConnection._conn.compareCPU(mox.IgnoreArg(),0).\ + AndRaise(libvirt.libvirtError('ERR')) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertRaises(libvirt.libvirtError, conn.compare_cpu, cpu_info) + self.mox.UnsetStubs() + + def test_compare_cpu_no_compatibility(self): + """libvirt.compare_cpu() return less than 0.(no compatibility)""" + + t = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ + """"topology":{"cores":"%s", "threads":"%s", """ + """"sockets":"%s"}, "features":[%s]}""") + cpu_info = t % ('x86', 'model', 'vendor', '2', '1', '4', '"tm"') + + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', + use_mock_anything=True) + libvirt_conn.LibvirtConnection._conn.compareCPU(mox.IgnoreArg(),0).\ + AndRaise(exception.Invalid('ERR')) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertRaises(exception.Invalid, conn.compare_cpu, cpu_info) + self.mox.UnsetStubs() + + def test_ensure_filtering_rules_for_instance_works_correctly(self): + """ensure_filtering_rules_for_instance works as expected correctly""" + + instance_ref = models.Instance() + instance_ref.__setitem__('id', 1) + + try: + nwmock, fwmock = self._driver_dependent_test_setup() + except: + return + + nwmock.setup_basic_filtering(mox.IgnoreArg()) + fwmock.prepare_instance_filter(instance_ref) + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', + use_mock_anything=True) + n = 'nova-instance-%s' % instance_ref.name + libvirt_conn.LibvirtConnection._conn.nwfilterLookupByName(n) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + conn.ensure_filtering_rules_for_instance(instance_ref) + self.mox.UnsetStubs() + + def test_ensure_filtering_rules_for_instance_timeout(self): + """ensure_filtering_fules_for_instance finishes with timeout""" + + instance_ref = models.Instance() + instance_ref.__setitem__('id', 1) + + try: + nwmock, fwmock = self._driver_dependent_test_setup() + except: + return + + nwmock.setup_basic_filtering(mox.IgnoreArg()) + fwmock.prepare_instance_filter(instance_ref) + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', + use_mock_anything=True) + n = 'nova-instance-%s' % instance_ref.name + for i in range(FLAGS.live_migration_timeout_sec * 2): + libvirt_conn.LibvirtConnection._conn.\ + nwfilterLookupByName(n).AndRaise(libvirt.libvirtError('ERR')) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + try: + conn.ensure_filtering_rules_for_instance(instance_ref) + except exception.Error, e: + c1 = ( 0<=e.message.find('Timeout migrating for')) + self.assertTrue(c1) + self.mox.UnsetStubs() + + def test_live_migration_works_correctly(self): + """_live_migration works as expected correctly """ + + class dummyCall(object): + f = None + def start(self, interval=0, now=False): + pass + + instance_ref = models.Instance() + instance_ref.__setitem__('id', 1) + dest = 'desthost' + ctxt = context.get_admin_context() + + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', + use_mock_anything=True) + vdmock = self.mox.CreateMock(libvirt.virDomain) + self.mox.StubOutWithMock(vdmock, "migrateToURI", + use_mock_anything=True) + vdmock.migrateToURI(FLAGS.live_migration_uri % dest, mox.IgnoreArg(), + None, FLAGS.live_migration_bandwidth).\ + AndReturn(None) + libvirt_conn.LibvirtConnection._conn.lookupByName(instance_ref.name).\ + AndReturn(vdmock) + # below description is also ok. + #self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection._conn, + # "lookupByName", use_mock_anything=True) + + libvirt_conn.utils.LoopingCall(f=None).AndReturn(dummyCall()) + + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + ret = conn._live_migration(ctxt, instance_ref, dest) + self.assertTrue(ret == None) + self.mox.UnsetStubs() + + def test_live_migration_raises_exception(self): + """ + _live_migration raises exception, then this testcase confirms + state_description/state for the instances/volumes are recovered. + """ + class Instance(models.NovaBase): + id = 0 + volumes = None + name = 'name' + + ctxt = context.get_admin_context() + dest = 'desthost' + instance_ref = Instance() + instance_ref.__setitem__('id', 1) + instance_ref.__setitem__('volumes', [{'id':1}, {'id':2}]) + + try: + nwmock, fwmock = self._driver_dependent_test_setup() + except: + return + + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', + use_mock_anything=True) + vdmock = self.mox.CreateMock(libvirt.virDomain) + self.mox.StubOutWithMock(vdmock, "migrateToURI", + use_mock_anything=True) + vdmock.migrateToURI(FLAGS.live_migration_uri % dest, mox.IgnoreArg(), + None, FLAGS.live_migration_bandwidth).\ + AndRaise(libvirt.libvirtError('ERR')) + libvirt_conn.LibvirtConnection._conn.lookupByName(instance_ref.name).\ + AndReturn(vdmock) + self.mox.StubOutWithMock(db, 'instance_set_state') + db.instance_set_state(ctxt, instance_ref['id'], + power_state.RUNNING, 'running') + self.mox.StubOutWithMock(db, 'volume_update') + for v in instance_ref.volumes: + db.volume_update(ctxt, v['id'], {'status': 'in-use'}).\ + InAnyOrder('g1') + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertRaises(libvirt.libvirtError, + conn._live_migration, + ctxt, instance_ref, dest) + self.mox.UnsetStubs() + + def test_post_live_migration_working_correctly(self): + """_post_live_migration works as expected correctly """ + + dest = 'dummydest' + ctxt = context.get_admin_context() + instance_ref = {'id':1, 'hostname':'i-00000001', 'host':dest, + 'fixed_ip':'dummyip', 'floating_ip':'dummyflip', + 'volumes':[{'id':1}, {'id':2} ]} + network_ref = {'id':1, 'host':dest} + floating_ip_ref = {'id':1, 'address':'1.1.1.1'} + + try: + nwmock, fwmock = self._driver_dependent_test_setup() + except: + return + fwmock.unfilter_instance(instance_ref) + + fixed_ip = instance_ref['fixed_ip'] + self.mox.StubOutWithMock(db, 'instance_get_fixed_address') + db.instance_get_fixed_address(ctxt, instance_ref['id']).AndReturn(fixed_ip) + self.mox.StubOutWithMock(db, 'fixed_ip_update') + db.fixed_ip_update(ctxt, fixed_ip, {'host': dest}) + self.mox.StubOutWithMock(db, 'fixed_ip_get_network') + db.fixed_ip_get_network(ctxt, fixed_ip).AndReturn(network_ref) + self.mox.StubOutWithMock(db, 'network_update') + db.network_update(ctxt, network_ref['id'], {'host': dest}) + + fl_ip = instance_ref['floating_ip'] + self.mox.StubOutWithMock(db, 'instance_get_floating_address') + db.instance_get_floating_address(ctxt, instance_ref['id']).AndReturn(fl_ip) + self.mox.StubOutWithMock(db, 'floating_ip_get_by_address') + db.floating_ip_get_by_address(ctxt, instance_ref['floating_ip']).\ + AndReturn(floating_ip_ref) + self.mox.StubOutWithMock(db, 'floating_ip_update') + db.floating_ip_update(ctxt, floating_ip_ref['address'], {'host': dest}) + + self.mox.StubOutWithMock(db, 'instance_update') + db.instance_update(ctxt, instance_ref['id'], + {'state_description': 'running', + 'state': power_state.RUNNING, 'host': dest}) + self.mox.StubOutWithMock(db, 'volume_update') + for v in instance_ref['volumes']: + db.volume_update(ctxt, v['id'], {'status': 'in-use'}) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + conn._post_live_migration( ctxt, instance_ref, dest) + self.mox.UnsetStubs() + + def test_post_live_migration_no_floating_ip(self): + """ + _post_live_migration works as expected correctly + (in case instance doesnt have floaitng ip) + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + instance_ref = {'id':1, 'hostname':'i-00000001', 'host':dest, + 'fixed_ip':'dummyip', 'floating_ip':'dummyflip', + 'volumes':[{'id':1}, {'id':2} ]} + network_ref = {'id':1, 'host':dest} + floating_ip_ref = {'id':1, 'address':'1.1.1.1'} + + try: + nwmock, fwmock = self._driver_dependent_test_setup() + except: + return + fwmock.unfilter_instance(instance_ref) + + fixed_ip = instance_ref['fixed_ip'] + self.mox.StubOutWithMock(db, 'instance_get_fixed_address') + db.instance_get_fixed_address(ctxt, instance_ref['id']).AndReturn(fixed_ip) + self.mox.StubOutWithMock(db, 'fixed_ip_update') + db.fixed_ip_update(ctxt, fixed_ip, {'host': dest}) + self.mox.StubOutWithMock(db, 'fixed_ip_get_network') + db.fixed_ip_get_network(ctxt, fixed_ip).AndReturn(network_ref) + self.mox.StubOutWithMock(db, 'network_update') + db.network_update(ctxt, network_ref['id'], {'host': dest}) + + self.mox.StubOutWithMock(db, 'instance_get_floating_address') + db.instance_get_floating_address(ctxt, instance_ref['id']).AndReturn(None) + self.mox.StubOutWithMock(libvirt_conn.LOG, 'info') + libvirt_conn.LOG.info(_('post livemigration operation is started..')) + libvirt_conn.LOG.info(_('floating_ip is not found for %s'), + instance_ref['hostname']) + # Checking last messages are ignored. may be no need to check so strictly? + libvirt_conn.LOG.info(mox.IgnoreArg()) + libvirt_conn.LOG.info(mox.IgnoreArg()) + + self.mox.StubOutWithMock(db, 'instance_update') + db.instance_update(ctxt, instance_ref['id'], + {'state_description': 'running', + 'state': power_state.RUNNING, + 'host': dest}) + self.mox.StubOutWithMock(db, 'volume_update') + for v in instance_ref['volumes']: + db.volume_update(ctxt, v['id'], {'status': 'in-use'}) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + conn._post_live_migration( ctxt, instance_ref, dest) + self.mox.UnsetStubs() + + def test_post_live_migration_no_floating_ip_with_exception(self): + """ + _post_live_migration works as expected correctly + (in case instance doesnt have floaitng ip, and raise exception) + """ + dest = 'dummydest' + ctxt = context.get_admin_context() + instance_ref = {'id':1, 'hostname':'i-00000001', 'host':dest, + 'fixed_ip':'dummyip', 'floating_ip':'dummyflip', + 'volumes':[{'id':1}, {'id':2} ]} + network_ref = {'id':1, 'host':dest} + floating_ip_ref = {'id':1, 'address':'1.1.1.1'} + + try: + nwmock, fwmock = self._driver_dependent_test_setup() + except: + return + fwmock.unfilter_instance(instance_ref) + + fixed_ip = instance_ref['fixed_ip'] + self.mox.StubOutWithMock(db, 'instance_get_fixed_address') + db.instance_get_fixed_address(ctxt, instance_ref['id']).AndReturn(fixed_ip) + self.mox.StubOutWithMock(db, 'fixed_ip_update') + db.fixed_ip_update(ctxt, fixed_ip, {'host': dest}) + self.mox.StubOutWithMock(db, 'fixed_ip_get_network') + db.fixed_ip_get_network(ctxt, fixed_ip).AndReturn(network_ref) + self.mox.StubOutWithMock(db, 'network_update') + db.network_update(ctxt, network_ref['id'], {'host': dest}) + + self.mox.StubOutWithMock(db, 'instance_get_floating_address') + db.instance_get_floating_address(ctxt, instance_ref['id']).\ + AndRaise(exception.NotFound()) + self.mox.StubOutWithMock(libvirt_conn.LOG, 'info') + libvirt_conn.LOG.info(_('post livemigration operation is started..')) + libvirt_conn.LOG.info(_('floating_ip is not found for %s'), + instance_ref['hostname']) + # the last message is ignored. may be no need to check so strictly? + libvirt_conn.LOG.info(mox.IgnoreArg()) + libvirt_conn.LOG.info(mox.IgnoreArg()) + + self.mox.StubOutWithMock(db, 'instance_update') + db.instance_update(ctxt, instance_ref['id'], + {'state_description': 'running', + 'state': power_state.RUNNING, 'host': dest}) + self.mox.StubOutWithMock(db, 'volume_update') + for v in instance_ref['volumes']: + db.volume_update(ctxt, v['id'], {'status': 'in-use'}) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + conn._post_live_migration( ctxt, instance_ref, dest) + self.mox.UnsetStubs() + def tearDown(self): super(LibvirtConnTestCase, self).tearDown() self.manager.delete_project(self.project) @@ -475,3 +988,4 @@ class NWFilterTestCase(test.TestCase): self.fw.prepare_instance_filter(instance) _ensure_all_called() self.teardown_security_group() + diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 80ae7f34c..f469af681 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -316,15 +316,15 @@ class FakeConnection(object): def get_vcpu_number(self): """This method is supported only libvirt. """ - return -1 + return def get_memory_mb(self): """This method is supported only libvirt..""" - return -1 + return def get_local_gb(self): """This method is supported only libvirt..""" - return -1 + return def get_hypervisor_type(self): """This method is supported only libvirt..""" @@ -332,12 +332,16 @@ class FakeConnection(object): def get_hypervisor_version(self): """This method is supported only libvirt..""" - return -1 + return def compare_cpu(self, xml): """This method is supported only libvirt..""" raise NotImplementedError('This method is supported only libvirt.') + def ensure_filtering_rules_for_instance(self, instance_ref): + """This method is supported only libvirt..""" + raise NotImplementedError('This method is supported only libvirt.') + def live_migration(self, context, instance_ref, dest): """This method is supported only libvirt..""" raise NotImplementedError('This method is supported only libvirt.') diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 7d1f76b32..49dd03c57 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -861,18 +861,18 @@ class LibvirtConnection(object): def get_cpu_info(self): """ Get cpuinfo information """ - xmlstr = self._conn.getCapabilities() - xml = libxml2.parseDoc(xmlstr) + xml = self._conn.getCapabilities() + xml = libxml2.parseDoc(xml) nodes = xml.xpathEval('//cpu') if len(nodes) != 1: - msg = 'Unexpected xml format. tag "cpu" must be 1, but %d.' \ - % len(nodes) + msg = 'Invalid xml. "" must be 1, but %d.' % len(nodes) msg += '\n' + xml.serialize() raise exception.Invalid(_(msg)) - arch = xml.xpathEval('//cpu/arch')[0].getContent() - model = xml.xpathEval('//cpu/model')[0].getContent() - vendor = xml.xpathEval('//cpu/vendor')[0].getContent() + cpu_info = dict() + cpu_info['arch'] = xml.xpathEval('//cpu/arch')[0].getContent() + cpu_info['model'] = xml.xpathEval('//cpu/model')[0].getContent() + cpu_info['vendor'] = xml.xpathEval('//cpu/vendor')[0].getContent() topology_node = xml.xpathEval('//cpu/topology')[0].get_properties() topology = dict() @@ -890,18 +890,19 @@ class LibvirtConnection(object): feature_nodes = xml.xpathEval('//cpu/feature') features = list() for nodes in feature_nodes: - feature_name = nodes.get_properties().getContent() - features.append(feature_name) + features.append(nodes.get_properties().getContent()) template = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ """"topology":{"cores":"%s", "threads":"%s", """ """"sockets":"%s"}, "features":[%s]}""") - c = topology['cores'] - s = topology['sockets'] - t = topology['threads'] f = ['"%s"' % x for x in features] - cpu_info = template % (arch, model, vendor, c, s, t, ', '.join(f)) - return cpu_info + return template % (cpu_info['arch'], + cpu_info['model'], + cpu_info['vendor'], + topology['cores'], + topology['sockets'], + topology['threads'], + ', '.join(f)) def block_stats(self, instance_name, disk): """ @@ -935,12 +936,12 @@ class LibvirtConnection(object): def compare_cpu(self, cpu_info): """ - Check the host cpu is compatible to a cpu given by xml. - "xml" must be a part of libvirt.openReadonly().getCapabilities(). - return values follows by virCPUCompareResult. - if 0 > return value, do live migration. + Check the host cpu is compatible to a cpu given by xml. + "xml" must be a part of libvirt.openReadonly().getCapabilities(). + return values follows by virCPUCompareResult. + if 0 > return value, do live migration. - 'http://libvirt.org/html/libvirt-libvirt.html#virCPUCompareResult' + 'http://libvirt.org/html/libvirt-libvirt.html#virCPUCompareResult' """ msg = _('Checking cpu_info: instance was launched this cpu.\n: %s ') LOG.info(msg % cpu_info) @@ -952,7 +953,7 @@ class LibvirtConnection(object): url = 'http://libvirt.org/html/libvirt-libvirt.html' url += '#virCPUCompareResult\n' msg = 'CPU does not have compativility.\n' - msg += 'result:%d \n' + msg += 'result:%s \n' msg += 'Refer to %s' msg = _(msg) @@ -960,7 +961,7 @@ class LibvirtConnection(object): try: ret = self._conn.compareCPU(xml, 0) except libvirt.libvirtError, e: - LOG.error(msg % (ret, url)) + LOG.error(msg % (e.message, url)) raise e if ret <= 0: @@ -969,24 +970,26 @@ class LibvirtConnection(object): return def ensure_filtering_rules_for_instance(self, instance_ref): - """ Setting up inevitable filtering rules on compute node, - and waiting for its completion. - To migrate an instance, filtering rules to hypervisors - and firewalls are inevitable on destination host. - ( Waiting only for filterling rules to hypervisor, - since filtering rules to firewall rules can be set faster). - - Concretely, the below method must be called. - - setup_basic_filtering (for nova-basic, etc.) - - prepare_instance_filter(for nova-instance-instance-xxx, etc.) - - to_xml may have to be called since it defines PROJNET, PROJMASK. - but libvirt migrates those value through migrateToURI(), - so , no need to be called. - - Don't use thread for this method since migration should - not be started when setting-up filtering rules operations - are not completed.""" + """ + Setting up inevitable filtering rules on compute node, + and waiting for its completion. + To migrate an instance, filtering rules to hypervisors + and firewalls are inevitable on destination host. + ( Waiting only for filterling rules to hypervisor, + since filtering rules to firewall rules can be set faster). + + Concretely, the below method must be called. + - setup_basic_filtering (for nova-basic, etc.) + - prepare_instance_filter(for nova-instance-instance-xxx, etc.) + + to_xml may have to be called since it defines PROJNET, PROJMASK. + but libvirt migrates those value through migrateToURI(), + so , no need to be called. + + Don't use thread for this method since migration should + not be started when setting-up filtering rules operations + are not completed. + """ # Tf any instances never launch at destination host, # basic-filtering must be set here. @@ -1009,40 +1012,44 @@ class LibvirtConnection(object): raise exception.Error(msg % (ec2_id, instance_ref.name)) time.sleep(0.5) - def live_migration(self, context, instance_ref, dest): + def live_migration(self, ctxt, instance_ref, dest): """ - Just spawning live_migration operation for - distributing high-load. + Just spawning live_migration operation for + distributing high-load. """ - greenthread.spawn(self._live_migration, context, instance_ref, dest) + greenthread.spawn(self._live_migration, ctxt, instance_ref, dest) - def _live_migration(self, context, instance_ref, dest): + def _live_migration(self, ctxt, instance_ref, dest): """ Do live migration.""" # Do live migration. try: - duri = FLAGS.live_migration_uri % dest - flaglist = FLAGS.live_migration_flag.split(',') flagvals = [getattr(libvirt, x.strip()) for x in flaglist] logical_sum = reduce(lambda x, y: x | y, flagvals) - bandwidth = FLAGS.live_migration_bandwidth - if self.read_only: tmpconn = self._connect(self.libvirt_uri, False) dom = tmpconn.lookupByName(instance_ref.name) - dom.migrateToURI(duri, logical_sum, None, bandwidth) + dom.migrateToURI(FLAGS.live_migration_uri % dest, + logical_sum, + None, + FLAGS.live_migration_bandwidth) tmpconn.close() else: dom = self._conn.lookupByName(instance_ref.name) - dom.migrateToURI(duri, logical_sum, None, bandwidth) + dom.migrateToURI(FLAGS.live_migration_uri % dest, + logical_sum, + None, + FLAGS.live_migration_bandwidth) except Exception, e: - id = instance_ref['id'] - db.instance_set_state(context, id, power_state.RUNNING, 'running') + db.instance_set_state(ctxt, + instance_ref['id'], + power_state.RUNNING, + 'running') for v in instance_ref['volumes']: - db.volume_update(context, + db.volume_update(ctxt, v['id'], {'status': 'in-use'}) @@ -1052,20 +1059,20 @@ class LibvirtConnection(object): timer = utils.LoopingCall(f=None) def wait_for_live_migration(): - + """waiting for live migration completion""" try: - state = self.get_info(instance_ref.name)['state'] + self.get_info(instance_ref.name)['state'] except exception.NotFound: timer.stop() - self._post_live_migration(context, instance_ref, dest) + self._post_live_migration(ctxt, instance_ref, dest) timer.f = wait_for_live_migration timer.start(interval=0.5, now=True) - def _post_live_migration(self, context, instance_ref, dest): + def _post_live_migration(self, ctxt, instance_ref, dest): """ - Post operations for live migration. - Mainly, database updating. + Post operations for live migration. + Mainly, database updating. """ LOG.info('post livemigration operation is started..') # Detaching volumes. @@ -1079,61 +1086,61 @@ class LibvirtConnection(object): 'nova.virt.libvirt_conn.IptablesFirewallDriver': try: self.firewall_driver.unfilter_instance(instance_ref) - except KeyError, e: + except KeyError: pass # Database updating. ec2_id = instance_ref['hostname'] instance_id = instance_ref['id'] - fixed_ip = db.instance_get_fixed_address(context, instance_id) + fixed_ip = db.instance_get_fixed_address(ctxt, instance_id) # Not return if fixed_ip is not found, otherwise, # instance never be accessible.. if None == fixed_ip: logging.warn('fixed_ip is not found for %s ' % ec2_id) - db.fixed_ip_update(context, fixed_ip, {'host': dest}) - network_ref = db.fixed_ip_get_network(context, fixed_ip) - db.network_update(context, network_ref['id'], {'host': dest}) + db.fixed_ip_update(ctxt, fixed_ip, {'host': dest}) + network_ref = db.fixed_ip_get_network(ctxt, fixed_ip) + db.network_update(ctxt, network_ref['id'], {'host': dest}) try: floating_ip \ - = db.instance_get_floating_address(context, instance_id) + = db.instance_get_floating_address(ctxt, instance_id) # Not return if floating_ip is not found, otherwise, # instance never be accessible.. if None == floating_ip: - logging.error('floating_ip is not found for %s ' % ec2_id) + LOG.info(_('floating_ip is not found for %s'), ec2_id) else: - floating_ip_ref = db.floating_ip_get_by_address(context, + floating_ip_ref = db.floating_ip_get_by_address(ctxt, floating_ip) - db.floating_ip_update(context, + db.floating_ip_update(ctxt, floating_ip_ref['address'], {'host': dest}) except exception.NotFound: - logging.debug('%s doesnt have floating_ip.. ' % ec2_id) + LOG.info(_('floating_ip is not found for %s'), ec2_id) except: - msg = 'Live migration: Unexpected error:' - msg += '%s cannot inherit floating ip.. ' % ec2_id - logging.error(_(msg)) + msg = ("""Live migration: Unexpected error:""" + """%s cannot inherit floating ip..""") + LOG.error(_(msg), ec2_id) # Restore instance/volume state - db.instance_update(context, + db.instance_update(ctxt, instance_id, {'state_description': 'running', 'state': power_state.RUNNING, 'host': dest}) for v in instance_ref['volumes']: - db.volume_update(context, + db.volume_update(ctxt, v['id'], {'status': 'in-use'}) - logging.info(_('Live migrating %s to %s finishes successfully') + LOG.info(_('Live migrating %s to %s finishes successfully') % (ec2_id, dest)) msg = _(("""Known error: the below error is nomally occurs.\n""" """Just check if iinstance is successfully migrated.\n""" """libvir: QEMU error : Domain not found: no domain """ """with matching name..""")) - logging.info(msg) + LOG.info(msg) class FirewallDriver(object): diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index c10f73fe7..1e7933f51 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -215,15 +215,15 @@ class XenAPIConnection(object): def get_vcpu_number(self): """This method is supported only libvirt. """ - return -1 + return def get_memory_mb(self): """This method is supported only libvirt..""" - return -1 + return def get_local_gb(self): """This method is supported only libvirt..""" - return -1 + return def get_hypervisor_type(self): """This method is supported only libvirt..""" @@ -231,12 +231,18 @@ class XenAPIConnection(object): def get_hypervisor_version(self): """This method is supported only libvirt..""" - return -1 + return def compare_cpu(self, xml): + """This method is supported only libvirt..""" + raise NotImplementedError('This method is supported only libvirt.') + + def ensure_filtering_rules_for_instance(self, instance_ref): + """This method is supported only libvirt..""" raise NotImplementedError('This method is supported only libvirt.') def live_migration(self, context, instance_ref, dest): + """This method is supported only libvirt..""" raise NotImplementedError('This method is supported only libvirt.') diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 1735d79eb..906eb86ea 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -153,6 +153,6 @@ class VolumeManager(manager.Manager): def check_for_export(self, context, instance_id): """Make sure whether volume is exported.""" if FLAGS.volume_driver == 'nova.volume.driver.AOEDriver': - instance_ref = self.db.instance_get(instance_id) + instance_ref = self.db.instance_get(context, instance_id) for v in instance_ref['volumes']: self.driver.check_for_export(context, v['id']) -- cgit From 09f2c4729456443c4874a8cadc53299817d6371a Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Mon, 31 Jan 2011 18:41:10 +0900 Subject: 1. Discard nova-manage host list Reason: nova-manage service list can be replacement. Changes: nova-manage 2. Fix resource checking inappropriate design. Reason: nova.scheduler.driver.has_enough_resource has inappropriate design, so fix it. This method didnt check free memory but check total memory. We need to register free memory onto databases(periodically). But periodically updating may causes flooding request to db in case of many compute-node. Currently, since memory information is only used in this feature, we take the choice that administrators manually has to execute nova-manage to let compute node update their own memory information. Changes: nova.db.sqlalchemy.models - Adding memory_mb_used, local_gb_used, vcpu_used column to Service. (local_gb and vcpu is just for reference to admins for now) nova.compute.manager - Changing nova.compute.manager.update_service Service table column is changed, so updating method must be changed. - Adding nova.compute.manager.update_available_resource a responder to admin's request to let compute nodes update their memory infomation nova.virt.libvirt_conn nova.virt.xenapi_conn nova.virt.fake - Adding getter method for memory_mb_used/local_gb_used/vcpu_used. nova-manage - request method to let compute nodes update their own memory info. --- bin/nova-manage | 92 ++++++++++++----------- nova/compute/manager.py | 54 ++++++++++++- nova/db/sqlalchemy/models.py | 5 +- nova/rpc.py | 3 + nova/scheduler/driver.py | 67 +++++++++++------ nova/scheduler/manager.py | 11 ++- nova/tests/test_compute.py | 55 ++++++++++++-- nova/tests/test_scheduler.py | 175 ++++++++++++++++++++++++++----------------- nova/tests/test_virt.py | 86 ++++++++++++++++++++- nova/utils.py | 18 +++++ nova/virt/fake.py | 22 ++++-- nova/virt/libvirt_conn.py | 37 +++++++-- nova/virt/xenapi_conn.py | 22 ++++-- 13 files changed, 476 insertions(+), 171 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 1ad3120b8..2831e273e 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -499,50 +499,6 @@ class InstanceCommands(object): print msg -class HostCommands(object): - """Class for mangaging host(physical nodes).""" - - def list(self): - """describe host list.""" - - # To supress msg: No handlers could be found for logger "amqplib" - logging.basicConfig() - - service_refs = db.service_get_all(context.get_admin_context()) - hosts = [h['host'] for h in service_refs] - hosts = list(set(hosts)) - for host in hosts: - print host - - def show(self, host): - """describe cpu/memory/hdd info for host.""" - - result = rpc.call(context.get_admin_context(), - FLAGS.scheduler_topic, - {"method": "show_host_resource", - "args": {"host": host}}) - - # Checking result msg format is necessary, that will have done - # when this feture is included in API. - if type(result) != dict: - print 'Unexpected error occurs' - elif not result['ret']: - print '%s' % result['msg'] - else: - cpu = result['phy_resource']['vcpus'] - mem = result['phy_resource']['memory_mb'] - hdd = result['phy_resource']['local_gb'] - - print 'HOST\t\tPROJECT\t\tcpu\tmem(mb)\tdisk(gb)' - print '%s\t\t\t%s\t%s\t%s' % (host, cpu, mem, hdd) - for p_id, val in result['usage'].items(): - print '%s\t%s\t\t%s\t%s\t%s' % (host, - p_id, - val['vcpus'], - val['memory_mb'], - val['local_gb']) - - class ServiceCommands(object): """Enable and disable running services""" @@ -587,6 +543,53 @@ class ServiceCommands(object): return db.service_update(ctxt, svc['id'], {'disabled': True}) + def describeresource(self, host): + """describe cpu/memory/hdd info for host.""" + + result = rpc.call(context.get_admin_context(), + FLAGS.scheduler_topic, + {"method": "show_host_resource", + "args": {"host": host}}) + + # Checking result msg format is necessary, that will have done + # when this feture is included in API. + if type(result) != dict: + print 'Unexpected error occurs' + elif not result['ret']: + print '%s' % result['msg'] + else: + cpu = result['phy_resource']['vcpus'] + mem = result['phy_resource']['memory_mb'] + hdd = result['phy_resource']['local_gb'] + cpu_u = result['phy_resource']['vcpus_used'] + mem_u = result['phy_resource']['memory_mb_used'] + hdd_u = result['phy_resource']['local_gb_used'] + + print 'HOST\t\t\tPROJECT\t\tcpu\tmem(mb)\tdisk(gb)' + print '%s(total)\t\t\t%s\t%s\t%s' % (host, cpu, mem, hdd) + print '%s(used)\t\t\t%s\t%s\t%s' % (host, cpu_u, mem_u, hdd_u) + for p_id, val in result['usage'].items(): + print '%s\t\t%s\t\t%s\t%s\t%s' % (host, + p_id, + val['vcpus'], + val['memory_mb'], + val['local_gb']) + + def updateresource(self, host): + """update available vcpu/memory/disk info for host.""" + + ctxt = context.get_admin_context() + service_refs = db.service_get_all_by_host(ctxt, host) + if len(service_refs) <= 0: + raise exception.Invalid(_('%s does not exists.') % host) + + service_refs = [s for s in service_refs if s['topic'] == 'compute'] + if len(service_refs) <= 0: + raise exception.Invalid(_('%s is not compute node.') % host) + + result = rpc.call(ctxt, db.queue_get_for(ctxt, FLAGS.compute_topic, host), + {"method": "update_available_resource"}) + class LogCommands(object): def request(self, request_id, logfile='/var/log/nova.log'): @@ -606,7 +609,6 @@ CATEGORIES = [ ('floating', FloatingIpCommands), ('network', NetworkCommands), ('instance', InstanceCommands), - ('host', HostCommands), ('service', ServiceCommands), ('log', LogCommands)] diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 4acba7153..e3c5d24b6 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -134,9 +134,12 @@ class ComputeManager(manager.Manager): raise exception.Invalid(msg) # Updating host information - vcpu = self.driver.get_vcpu_number() - memory_mb = self.driver.get_memory_mb() - local_gb = self.driver.get_local_gb() + vcpu = self.driver.get_vcpu_total() + memory_mb = self.driver.get_memory_mb_total() + local_gb = self.driver.get_local_gb_total() + vcpu_u = self.driver.get_vcpu_used() + memory_mb_u = self.driver.get_memory_mb_used() + local_gb_u = self.driver.get_local_gb_used() hypervisor = self.driver.get_hypervisor_type() version = self.driver.get_hypervisor_version() cpu_info = self.driver.get_cpu_info() @@ -146,10 +149,42 @@ class ComputeManager(manager.Manager): {'vcpus': vcpu, 'memory_mb': memory_mb, 'local_gb': local_gb, + 'vcpus_used':vcpu_u, + 'memory_mb_used': memory_mb_u, + 'local_gb_used': local_gb_u, 'hypervisor_type': hypervisor, 'hypervisor_version': version, 'cpu_info': cpu_info}) + def update_available_resource(self, context): + """ + update compute node specific info to DB. + Alghough this might be subset of update_service, + udpate_service() is used only nova-compute is lauched. + On the other hand, this method is used whenever administrators + request comes. + """ + try: + service_ref = self.db.service_get_by_args(context, + self.host, + 'nova-compute') + except exception.NotFound: + msg = _(("""Cannot update resource info.""" + """ Because no service record found.""")) + raise exception.Invalid(msg) + + # Updating host information + vcpu_u = self.driver.get_vcpu_used() + memory_mb_u = self.driver.get_memory_mb_used() + local_gb_u = self.driver.get_local_gb_used() + + self.db.service_update(context, + service_ref['id'], + {'vcpus_used':vcpu_u, + 'memory_mb_used': memory_mb_u, + 'local_gb_used': local_gb_u}) + return + def _update_state(self, context, instance_id): """Update the state of an instance from the driver info.""" # FIXME(ja): include other fields from state? @@ -596,6 +631,19 @@ class ComputeManager(manager.Manager): """ Check the host cpu is compatible to a cpu given by xml.""" return self.driver.compare_cpu(cpu_info) + def mktmpfile(self, context): + """make tmpfile under FLAGS.instance_path.""" + return utils.mktmpfile(FLAGS.instances_path) + + def exists(self, context, path): + """Confirm existence of the tmpfile given by path.""" + if not utils.exists(path): + raise exception.NotFound(_('%s not found') % path) + + def remove(self, context, path): + """remove the tmpfile given by path.""" + return utils.remove(path) + def pre_live_migration(self, context, instance_id): """Any preparation for live migration at dst host.""" diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 7c40d5596..217b14bf7 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -164,7 +164,10 @@ class Service(BASE, NovaBase): vcpus = Column(Integer, nullable=True) memory_mb = Column(Integer, nullable=True) local_gb = Column(Integer, nullable=True) - hypervisor_type = Column(String(128), nullable=True) + vcpus_used = Column(Integer, nullable=True) + memory_mb_used = Column(Integer, nullable=True) + local_gb_used = Column(Integer, nullable=True) + hypervisor_type = Column(Text(), nullable=True) hypervisor_version = Column(Integer, nullable=True) # Note(masumotok): Expected Strings example: # diff --git a/nova/rpc.py b/nova/rpc.py index 49b11602b..cf4004079 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -250,6 +250,9 @@ def msg_reply(msg_id, reply=None, failure=None): try: publisher.send({'result': reply, 'failure': failure}) except TypeError: + print '>>>>>>>>>>>>>>>>>>' + print reply + print '>>>>>>>>>>>>>>>>>>' publisher.send( {'result': dict((k, repr(v)) for k, v in reply.__dict__.iteritems()), diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index d4ad42388..937f09c6f 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -33,6 +33,7 @@ from nova.compute import power_state FLAGS = flags.FLAGS flags.DEFINE_integer('service_down_time', 60, 'maximum time since last checkin for up service') +flags.DECLARE('instances_path', 'nova.compute.manager') class NoValidHost(exception.Error): @@ -163,6 +164,8 @@ class Scheduler(object): http://wiki.libvirt.org/page/TodoPreMigrationChecks """ + # Checking shared storage connectivity + self.mounted_on_same_shared_storage(context, instance_ref, dest) # Checking dest exists. dservice_refs = db.service_get_all_by_host(context, dest) @@ -207,38 +210,60 @@ class Scheduler(object): raise e def has_enough_resource(self, context, instance_ref, dest): - """Check if destination host has enough resource for live migration""" + """ + Check if destination host has enough resource for live migration. + Currently, only memory checking has been done. + If storage migration(block migration, meaning live-migration + without any shared storage) will be available, local storage + checking is also necessary. + """ # Getting instance information ec2_id = instance_ref['hostname'] - vcpus = instance_ref['vcpus'] mem = instance_ref['memory_mb'] - hdd = instance_ref['local_gb'] - # Gettin host information + # Getting host information service_refs = db.service_get_all_by_host(context, dest) if len(service_refs) <= 0: raise exception.Invalid(_('%s does not exists.') % dest) service_ref = service_refs[0] - total_cpu = int(service_ref['vcpus']) - total_mem = int(service_ref['memory_mb']) - total_hdd = int(service_ref['local_gb']) + mem_total = int(service_ref['memory_mb']) + mem_used = int(service_ref['memory_mb_used']) + mem_avail = mem_total - mem_used + mem_inst = instance_ref['memory_mb'] + if mem_avail <= mem_inst: + msg = _('%s is not capable to migrate %s(host:%s <= instance:%s)') + raise exception.NotEmpty(msg % (dest, ec2_id, mem_avail, mem_inst)) - instances_refs = db.instance_get_all_by_host(context, dest) - for i_ref in instances_refs: - total_cpu -= int(i_ref['vcpus']) - total_mem -= int(i_ref['memory_mb']) - total_hdd -= int(i_ref['local_gb']) + def mounted_on_same_shared_storage(self, context, instance_ref, dest): + """ + Check if /nova-inst-dir/insntances is mounted same storage at + live-migration src and dest host. + """ + src = instance_ref['host'] + dst_t = db.queue_get_for(context, FLAGS.compute_topic, dest) + src_t = db.queue_get_for(context, FLAGS.compute_topic, src) - # Checking host has enough information - logging.debug(_('host(%s) remains vcpu:%s mem:%s hdd:%s,') % - (dest, total_cpu, total_mem, total_hdd)) - logging.debug(_('instance(%s) has vcpu:%s mem:%s hdd:%s,') % - (ec2_id, vcpus, mem, hdd)) + # create tmpfile at dest host + try: + filename = rpc.call(context, dst_t, {"method": 'mktmpfile'}) + except rpc.RemoteError, e: + msg = _("Cannot create tmpfile at %s to confirm shared storage.") + logging.error(msg % FLAGS.instance_path) + raise e - if total_cpu <= vcpus or total_mem <= mem or total_hdd <= hdd: - raise exception.NotEmpty(_('%s is not capable to migrate %s') % - (dest, ec2_id)) + # make sure existence at src host. + try: + rpc.call(context, src_t, + {"method": 'exists', "args":{'path':filename}}) + + except (rpc.RemoteError, exception.NotFound), e: + msg = (_("""Cannot comfirm %s at %s to confirm shared storage.""" + """Check if %s is same shared storage""")) + logging.error(msg % FLAGS.instance_path) + raise e - logging.debug(_('%s has_enough_resource() for %s') % (dest, ec2_id)) + # then remove. + rpc.call(context, dst_t, + {"method": 'remove', "args":{'path':filename}}) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index a181225a6..b40f46a85 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -84,7 +84,10 @@ class SchedulerManager(manager.Manager): # Getting physical resource information h_resource = {'vcpus': service_ref['vcpus'], 'memory_mb': service_ref['memory_mb'], - 'local_gb': service_ref['local_gb']} + 'local_gb': service_ref['local_gb'], + 'vcpus_used': service_ref['vcpus_used'], + 'memory_mb_used': service_ref['memory_mb_used'], + 'local_gb_used': service_ref['local_gb_used']} # Getting usage resource information u_resource = {} @@ -108,8 +111,8 @@ class SchedulerManager(manager.Manager): hdd = db.instance_get_disk_sum_by_host_and_project(context, host, p_id) - u_resource[p_id] = {'vcpus': vcpus, - 'memory_mb': mem, - 'local_gb': hdd} + u_resource[p_id] = {'vcpus': int(vcpus), + 'memory_mb': int(mem), + 'local_gb': int(hdd)} return {'ret': True, 'phy_resource': h_resource, 'usage': u_resource} diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 344c2d2b5..8d3ac315d 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -268,7 +268,8 @@ class ComputeTestCase(test.TestCase): """ def dic_key_check(dic): - validkey = ['vcpus', 'memory_mb', 'local_gb', + validkey = ['vcpus', 'memory_mb', 'local_gb', + 'vcpus_used', 'memory_mb_used', 'local_gb_used', 'hypervisor_type', 'hypervisor_version', 'cpu_info'] return (list(set(validkey)) == list(set(dic.keys()))) @@ -286,13 +287,55 @@ class ComputeTestCase(test.TestCase): self.compute.db = dbmock self.mox.ReplayAll() + self.compute.update_service('dummy', host, binary) + self.mox.ResetAll() + + def test_update_available_resource_exception(self): + """a testcase of update_available_resource raises exception""" + host = 'foo' + binary = 'nova-compute' + ctxt = context.get_admin_context() + dbmock = self.mox.CreateMock(db) + dbmock.service_get_by_args(mox.IgnoreArg(), + mox.StrContains(host), + mox.StrContains(binary)).\ + AndRaise(exception.NotFound()) + self.compute.db = dbmock + self.compute.host = host + self.mox.ReplayAll() try: - self.compute.update_service('dummy', host, binary) + self.compute.update_available_resource(ctxt) except exception.Invalid, e: - msg = 'Cannot insert compute manager specific info' + msg = 'Cannot update resource info.' c1 = ( 0 <= e.message.find(msg)) self.assertTrue(c1) - self.mox.ResetAll() + self.mox.UnsetStubs() + + def test_update_available_resource_success(self): + """a testcase of update_available_resource finishes with no errors""" + + def dic_key_check(dic): + validkey = [ 'vcpus_avail', 'memory_mb_avail', 'local_gb_avail'] + return (list(set(validkey)) == list(set(dic.keys()))) + + host = 'foo' + binary = 'nova-compute' + ctxt = context.get_admin_context() + service_ref = {'id':1, 'binary':'nova-compute', 'topic':'compute'} + dbmock = self.mox.CreateMock(db) + dbmock.service_get_by_args(mox.IgnoreArg(), + mox.StrContains(host), + mox.StrContains(binary)).\ + AndReturn(service_ref) + dbmock.service_update(mox.IgnoreArg(), + service_ref['id'], + mox.Func(dic_key_check)) + + self.compute.db = dbmock + self.compute.host = host + self.mox.ReplayAll() + self.compute.update_available_resource(ctxt) + self.mox.UnsetStubs() def _setup_other_managers(self): self.volume_manager = utils.import_object(FLAGS.volume_manager) @@ -444,7 +487,7 @@ class ComputeTestCase(test.TestCase): rpc.call(c, db.queue_get_for(c, FLAGS.compute_topic, dest), {"method": "pre_live_migration", "args": {'instance_id': i_id}}).\ - InAnyOrder('g1').AndRaise(rpc.RemoteError('du', 'mm', 'y')) + InAnyOrder('g1').AndRaise(rpc.RemoteError('', '', '')) self.mox.StubOutWithMock(compute_manager.LOG, 'error') compute_manager.LOG.error('Pre live migration for %s failed at %s', instance_ref['hostname'], dest) @@ -480,7 +523,7 @@ class ComputeTestCase(test.TestCase): rpc.call(c, compute_topic, {"method": "pre_live_migration", "args": {'instance_id': i_id}}).\ - AndRaise(rpc.RemoteError('du', 'mm', 'y')) + AndRaise(rpc.RemoteError('', '', '')) self.mox.StubOutWithMock(compute_manager.LOG, 'error') compute_manager.LOG.error('Pre live migration for %s failed at %s', instance_ref['hostname'], dest) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index c62bca9b1..36d99d666 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -41,6 +41,7 @@ from nova.db.sqlalchemy import models FLAGS = flags.FLAGS flags.DECLARE('max_cores', 'nova.scheduler.simple') flags.DECLARE('stub_network', 'nova.compute.manager') +flags.DECLARE('instances_path', 'nova.compute.manager') class TestDriver(driver.Scheduler): @@ -111,7 +112,8 @@ class SchedulerTestCase(test.TestCase): scheduler = manager.SchedulerManager() dest = 'dummydest' ctxt = context.get_admin_context() - r0 = {'vcpus':16, 'memory_mb':32, 'local_gb':100} + r0 = {'vcpus':16, 'memory_mb':32, 'local_gb':100, + 'vcpus_used':16, 'memory_mb_used':32, 'local_gb_used':10} service_ref = {'id':1, 'host':dest} service_ref.update(r0) @@ -140,7 +142,8 @@ class SchedulerTestCase(test.TestCase): scheduler = manager.SchedulerManager() dest = 'dummydest' ctxt = context.get_admin_context() - r0 = {'vcpus':16, 'memory_mb':32, 'local_gb':100} + r0 = {'vcpus':16, 'memory_mb':32, 'local_gb':100, + 'vcpus_used':16, 'memory_mb_used':32, 'local_gb_used':10} r1 = {'vcpus':10, 'memory_mb':4, 'local_gb':20} r2 = {'vcpus':10, 'memory_mb':20, 'local_gb':30} service_ref = {'id':1, 'host':dest} @@ -148,7 +151,7 @@ class SchedulerTestCase(test.TestCase): instance_ref2 = {'id':2, 'project_id':'p-01', 'host':'dummy'} instance_ref2.update(r1) instance_ref3 = {'id':3, 'project_id':'p-02', 'host':'dummy'} - instance_ref3.update(r1) + instance_ref3.update(r2) self.mox.StubOutWithMock(manager, 'db', use_mock_anything=True) manager.db.service_get_all_compute_sorted(mox.IgnoreArg()).\ @@ -176,6 +179,7 @@ class SchedulerTestCase(test.TestCase): self.assertTrue( c1 and c2 and c3 and c4 and c5 and c6) self.mox.UnsetStubs() + class ZoneSchedulerTestCase(test.TestCase): """Test case for zone scheduler""" def setUp(self): @@ -495,7 +499,7 @@ class SimpleDriverTestCase(test.TestCase): ctxt = context.get_admin_context() topic = FLAGS.compute_topic i_ref = {'id':1, 'hostname':'i-00000001', 'host':'dummy', - 'volumes':[{'id':1}, {'id':2}]} + 'volumes':[{'id':1}, {'id':2}]} dest = 'dummydest' self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) @@ -793,7 +797,10 @@ class SimpleDriverTestCase(test.TestCase): ctxt = context.get_admin_context() topic = FLAGS.compute_topic i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + driver_i = self.scheduler.driver + self.mox.StubOutWithMock(driver_i, 'mounted_on_same_shared_storage') + driver_i.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([]) @@ -813,6 +820,7 @@ class SimpleDriverTestCase(test.TestCase): Original host(an instance launched on) does not exist. """ dest = 'dummydest' + driver_i = self.scheduler.driver ctxt = context.get_admin_context() topic = FLAGS.compute_topic i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', 'launched_on':'h1'} @@ -821,6 +829,8 @@ class SimpleDriverTestCase(test.TestCase): service_ref.__setitem__('topic', 'compute') service_ref.__setitem__('host', i_ref['host']) + self.mox.StubOutWithMock(driver_i, 'mounted_on_same_shared_storage') + driver_i.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([service_ref]) @@ -844,6 +854,7 @@ class SimpleDriverTestCase(test.TestCase): Original host and dest host has different hypervisor type. """ dest = 'dummydest' + driver_i = self.scheduler.driver ctxt = context.get_admin_context() topic = FLAGS.compute_topic i_ref = {'id':1, 'hostname':'i-01', @@ -856,6 +867,8 @@ class SimpleDriverTestCase(test.TestCase): service_ref2.__setitem__('id', 2) service_ref2.__setitem__('hypervisor_type', 'xen') + self.mox.StubOutWithMock(driver_i, 'mounted_on_same_shared_storage') + driver_i.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([service_ref]) @@ -879,6 +892,7 @@ class SimpleDriverTestCase(test.TestCase): Original host and dest host has different hypervisor version. """ dest = 'dummydest' + driver_i = self.scheduler.driver ctxt = context.get_admin_context() topic = FLAGS.compute_topic i_ref = {'id':1, 'hostname':'i-01', @@ -891,6 +905,8 @@ class SimpleDriverTestCase(test.TestCase): service_ref2.__setitem__('id', 2) service_ref2.__setitem__('hypervisor_version', 12001) + self.mox.StubOutWithMock(driver_i, 'mounted_on_same_shared_storage') + driver_i.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([service_ref]) @@ -914,6 +930,7 @@ class SimpleDriverTestCase(test.TestCase): Original host and dest host has different hypervisor version. """ dest = 'dummydest' + driver_i = self.scheduler.driver ctxt = context.get_admin_context() topic = FLAGS.compute_topic i_ref = {'id':1, 'hostname':'i-01', @@ -927,6 +944,8 @@ class SimpleDriverTestCase(test.TestCase): service_ref2.__setitem__('hypervisor_version', 12000) service_ref2.__setitem__('cpuinfo', 'info') + self.mox.StubOutWithMock(driver_i, 'mounted_on_same_shared_storage') + driver_i.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([service_ref]) @@ -956,6 +975,7 @@ class SimpleDriverTestCase(test.TestCase): The testcase make sure everything finished with no error. """ dest = 'dummydest' + driver_i = self.scheduler.driver ctxt = context.get_admin_context() topic = FLAGS.compute_topic i_ref = {'id':1, 'hostname':'i-01', @@ -969,6 +989,8 @@ class SimpleDriverTestCase(test.TestCase): service_ref2.__setitem__('hypervisor_version', 12000) service_ref2.__setitem__('cpuinfo', 'info') + self.mox.StubOutWithMock(driver_i, 'mounted_on_same_shared_storage') + driver_i.mounted_on_same_shared_storage(mox.IgnoreArg(), i_ref, dest) self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([service_ref]) @@ -988,121 +1010,134 @@ class SimpleDriverTestCase(test.TestCase): self.assertTrue(ret == None) self.mox.UnsetStubs() - def test_has_enough_resource_lack_resource_vcpu(self): + def test_has_enough_resource_lack_resource_memory(self): """ A testcase of driver.has_enough_resource. - Lack of vcpu.(boundary check) + Lack of memory_mb.(boundary check) """ dest = 'dummydest' ctxt = context.get_admin_context() topic = FLAGS.compute_topic - service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} - i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', - 'vcpus':6, 'memory_mb':8, 'local_gb':10} - i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} - i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} + service_ref = {'id':1, 'memory_mb':32, 'memory_mb_used':12, 'local_gb':100} + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', + 'vcpus':5, 'memory_mb':20, 'local_gb':10} self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([service_ref]) - driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ - AndReturn([i_ref2, i_ref3]) self.mox.ReplayAll() try: - self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) + self.scheduler.driver.has_enough_resource(ctxt, i_ref, dest) except exception.NotEmpty, e: msg = 'is not capable to migrate' self.assertTrue(e.message.find(msg) >= 0) self.mox.UnsetStubs() + self.mox.UnsetStubs() - def test_has_enough_resource_lack_resource_memory(self): + def test_has_enough_resource_works_correctly(self): """ - A testcase of driver.has_enough_resource. - Lack of memory_mb.(boundary check) + A testcase of driver.has_enough_resource + to make sure everything finished with no error. """ dest = 'dummydest' ctxt = context.get_admin_context() topic = FLAGS.compute_topic - service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} - i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', - 'vcpus':5, 'memory_mb':16, 'local_gb':10} - i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} - i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', + service_ref = {'id':1, 'memory_mb':120, 'memory_mb_used':32} + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy', 'vcpus':5, 'memory_mb':8, 'local_gb':10} self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ AndReturn([service_ref]) - driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ - AndReturn([i_ref2, i_ref3]) self.mox.ReplayAll() - try: - self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) - except exception.NotEmpty, e: - msg = 'is not capable to migrate' - self.assertTrue(e.message.find(msg) >= 0) + ret = self.scheduler.driver.has_enough_resource(ctxt, i_ref, dest) + self.assertTrue(ret == None) self.mox.UnsetStubs() + + def test_mounted_on_same_shared_storage_cannot_make_tmpfile(self): + """ + A testcase of driver.mounted_on_same_shared_storage + checks log message when dest host cannot make tmpfile. + """ + dest = 'dummydest' + driver_i = self.scheduler.driver + ctxt = context.get_admin_context() + topic = FLAGS.compute_topic + fpath = '/test/20110127120000' + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + + self.mox.StubOutWithMock(driver, 'rpc', use_mock_anything=True) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(ctxt, FLAGS.compute_topic, dest), + {"method": 'mktmpfile'}).AndRaise(rpc.RemoteError('', '', '')) + self.mox.StubOutWithMock(driver.logging, 'error') + msg = _("Cannot create tmpfile at %s to confirm shared storage.") + driver.logging.error(msg % FLAGS.instances_path) + + self.mox.ReplayAll() + self.assertRaises(rpc.RemoteError, + driver_i.mounted_on_same_shared_storage, + ctxt, i_ref, dest) self.mox.UnsetStubs() - def test_has_enough_resource_lack_resource_disk(self): + def test_mounted_on_same_shared_storage_cannot_comfirm_tmpfile(self): """ - A testcase of driver.has_enough_resource. - Lack of local_gb.(boundary check) + A testcase of driver.mounted_on_same_shared_storage + checks log message when src host cannot comfirm tmpfile. """ - scheduler = manager.SchedulerManager() dest = 'dummydest' + driver_i = self.scheduler.driver ctxt = context.get_admin_context() topic = FLAGS.compute_topic - service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} - i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':80} - i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} - i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} + fpath = '/test/20110127120000' + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + + self.mox.StubOutWithMock(driver, 'rpc', use_mock_anything=True) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(ctxt, FLAGS.compute_topic, dest), + {"method": 'mktmpfile'}).AndReturn(fpath) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(ctxt, FLAGS.compute_topic, i_ref['host']), + {"method": 'exists', "args":{'path':fpath}}).\ + AndRaise(rpc.RemoteError('','','')) + self.mox.StubOutWithMock(driver.logging, 'error') + msg = _("Cannot create tmpfile at %s to confirm shared storage.") + driver.logging.error(msg % FLAGS.instances_path) - self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) - driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ - AndReturn([service_ref]) - driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ - AndReturn([i_ref2, i_ref3]) - self.mox.ReplayAll() - try: - self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) - except exception.NotEmpty, e: - msg = 'is not capable to migrate' - self.assertTrue(e.message.find(msg) >= 0) + self.assertRaises(rpc.RemoteError, + driver_i.mounted_on_same_shared_storage, + ctxt, i_ref, dest) self.mox.UnsetStubs() - def test_has_enough_resource_works_correctly(self): + + def test_mounted_on_same_shared_storage_works_correctly(self): """ - A testcase of driver.has_enough_resource + A testcase of driver.mounted_on_same_shared_storage to make sure everything finished with no error. """ dest = 'dummydest' ctxt = context.get_admin_context() topic = FLAGS.compute_topic - service_ref = {'id':1, 'vcpus':16, 'memory_mb':32, 'local_gb':100} - i_ref1 = {'id':1, 'hostname':'i-01', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} - i_ref2 = {'id':2, 'hostname':'i-01', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} - i_ref3 = {'id':3, 'hostname':'i-02', 'host':'dummy', - 'vcpus':5, 'memory_mb':8, 'local_gb':10} + fpath = '/test/20110127120000' + i_ref = {'id':1, 'hostname':'i-01', 'host':'dummy'} + + self.mox.StubOutWithMock(driver, 'rpc', use_mock_anything=True) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(mox.IgnoreArg(), FLAGS.compute_topic, dest), + {"method": 'mktmpfile'}).AndReturn(fpath) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(mox.IgnoreArg(), FLAGS.compute_topic, i_ref['host']), + {"method": 'exists', "args":{'path':fpath}}) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(mox.IgnoreArg(), FLAGS.compute_topic, dest), + {"method": 'remove', "args":{'path':fpath}}) - self.mox.StubOutWithMock(driver, 'db', use_mock_anything=True) - driver.db.service_get_all_by_host(mox.IgnoreArg(), dest).\ - AndReturn([service_ref]) - driver.db.instance_get_all_by_host(mox.IgnoreArg(), dest).\ - AndReturn([i_ref2, i_ref3]) - self.mox.ReplayAll() - ret = self.scheduler.driver.has_enough_resource(ctxt, i_ref1, dest) + ret = self.scheduler.driver.mounted_on_same_shared_storage(ctxt, + i_ref, + dest) self.assertTrue(ret == None) self.mox.UnsetStubs() diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 177e8f021..2828baced 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -241,11 +241,11 @@ class LibvirtConnTestCase(test.TestCase): uri = conn.get_uri() self.assertEquals(uri, testuri) - def test_get_memory_mb(self): + def test_get_vcpu_total(self): """ - Check if get_memory_mb returns memory value + Check if get_vcpu_total returns appropriate cpu value Connection/OS/driver differenct does not matter for this method, - so everyone can execute for checking. + everyone can execute for checking. """ try: self._driver_dependent_test_setup() @@ -254,9 +254,87 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) - self.assertTrue(0 < conn.get_memory_mb()) + self.assertTrue(0 < conn.get_vcpu_total()) self.mox.UnsetStubs() + + def test_get_memory_mb_total(self): + """Check if get_memory_mb returns appropriate memory value""" + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertTrue(0 < conn.get_memory_mb_total()) + self.mox.UnsetStubs() + + def test_get_local_gb_total(self): + """Check if get_local_gb_total returns appropriate disk value""" + # Note(masumotok): cannot test b/c FLAGS.instances_path is + # inevitable for this test.. + #try: + # self._driver_dependent_test_setup() + #except: + # return + # + #self.mox.ReplayAll() + #conn = libvirt_conn.LibvirtConnection(False) + #self.assertTrue(0 < conn.get_local_gb_total()) + #self.mox.UnsetStubs() + pass + + def test_get_vcpu_used(self): + """Check if get_local_gb_total returns appropriate disk value""" + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn', use_mock_anything=True) + libvirt_conn.LibvirtConnection._conn.listDomainsID().AndReturn([1,2]) + vdmock = self.mox.CreateMock(libvirt.virDomain) + self.mox.StubOutWithMock(vdmock, "vcpus", use_mock_anything=True) + vdmock.vcpus().AndReturn(['', [('dummycpu'), ('dummycpu')]]) + vdmock.vcpus().AndReturn(['', [('dummycpu'), ('dummycpu')]]) + libvirt_conn.LibvirtConnection._conn.lookupByID(mox.IgnoreArg()).\ + AndReturn(vdmock) + libvirt_conn.LibvirtConnection._conn.lookupByID(mox.IgnoreArg()).\ + AndReturn(vdmock) + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertTrue( conn.get_vcpu_used() == 4) + self.mox.UnsetStubs() + + def test_get_memory_mb_used(self): + """Check if get_memory_mb returns appropriate memory value""" + try: + self._driver_dependent_test_setup() + except: + return + + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertTrue(0 < conn.get_memory_mb_used()) + self.mox.UnsetStubs() + + def test_get_local_gb_used(self): + """Check if get_local_gb_total returns appropriate disk value""" + # Note(masumotok): cannot test b/c FLAGS.instances_path is + # inevitable for this test.. + #try: + # self._driver_dependent_test_setup() + #except: + # return + + #self.mox.ReplayAll() + #conn = libvirt_conn.LibvirtConnection(False) + #self.assertTrue(0 < conn.get_local_gb_used()) + #self.mox.UnsetStubs() + pass + def test_get_cpu_info_works_correctly(self): """ Check if get_cpu_info works correctly. diff --git a/nova/utils.py b/nova/utils.py index 6d3ddd092..e8919d7a5 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -303,6 +303,24 @@ def str_dict_replace(s, mapping): return s +def mktmpfile(dir): + """create tmpfile under dir, and return filename.""" + filename = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S') + fpath = os.path.join(dir, filename) + open(fpath, 'a+').write(fpath+'\n') + return fpath + + +def exists(filename): + """check file path existence.""" + return os.path.exists(filename) + + +def remove(filename): + """remove file.""" + return os.remove(filename) + + class LazyPluggable(object): """A pluggable backend loaded lazily based on some value.""" diff --git a/nova/virt/fake.py b/nova/virt/fake.py index f469af681..4bf477f5b 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -314,16 +314,28 @@ class FakeConnection(object): """This method is supported only libvirt. """ return - def get_vcpu_number(self): + def get_vcpu_total(self): """This method is supported only libvirt. """ return - def get_memory_mb(self): - """This method is supported only libvirt..""" + def get_memory_mb_total(self): + """This method is supported only libvirt. """ return - def get_local_gb(self): - """This method is supported only libvirt..""" + def get_local_gb_total(self): + """This method is supported only libvirt. """ + return + + def get_vcpu_used(self): + """This method is supported only libvirt. """ + return + + def get_memory_mb_used(self): + """This method is supported only libvirt. """ + return + + def get_local_gb_used(self): + """This method is supported only libvirt. """ return def get_hypervisor_type(self): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 49dd03c57..aefa32dcb 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -835,21 +835,44 @@ class LibvirtConnection(object): return interfaces - def get_vcpu_number(self): + def get_vcpu_total(self): """ Get vcpu number of physical computer. """ - return self._conn.getMaxVcpus(None) + return open('/proc/cpuinfo').read().count('processor') - def get_memory_mb(self): - """Get the memory size of physical computer .""" + def get_memory_mb_total(self): + """Get the total memory size(MB) of physical computer .""" meminfo = open('/proc/meminfo').read().split() idx = meminfo.index('MemTotal:') # transforming kb to mb. return int(meminfo[idx + 1]) / 1024 - def get_local_gb(self): - """Get the hdd size of physical computer .""" + def get_local_gb_total(self): + """Get the total hdd size(GB) of physical computer .""" hddinfo = os.statvfs(FLAGS.instances_path) - return hddinfo.f_bsize * hddinfo.f_blocks / 1024 / 1024 / 1024 + return hddinfo.f_frsize * hddinfo.f_blocks / 1024 / 1024 / 1024 + + def get_vcpu_used(self): + """ Get vcpu available number of physical computer. """ + total = 0 + for i in self._conn.listDomainsID(): + dom = self._conn.lookupByID(i) + total += len(dom.vcpus()[1]) + return total + + def get_memory_mb_used(self): + """Get the free memory size(MB) of physical computer.""" + m = open('/proc/meminfo').read().split() + idx1 = m.index('MemFree:') + idx2 = m.index('Buffers:') + idx3 = m.index('Cached:') + avail = (int(m[idx1+1]) + int(m[idx2+1]) + int(m[idx3+1])) / 1024 + return self.get_memory_mb_total() - avail + + def get_local_gb_used(self): + """Get the free hdd size(GB) of physical computer .""" + hddinfo = os.statvfs(FLAGS.instances_path) + avail = hddinfo.f_frsize * hddinfo.f_bavail / 1024 / 1024 / 1024 + return self.get_local_gb_total() - avail def get_hypervisor_type(self): """ Get hypervisor type """ diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 1e7933f51..902879d09 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -213,16 +213,28 @@ class XenAPIConnection(object): """This method is supported only libvirt. """ return - def get_vcpu_number(self): + def get_vcpu_total(self): """This method is supported only libvirt. """ return - def get_memory_mb(self): - """This method is supported only libvirt..""" + def get_memory_mb_total(self): + """This method is supported only libvirt. """ return - def get_local_gb(self): - """This method is supported only libvirt..""" + def get_local_gb_total(self): + """This method is supported only libvirt. """ + return + + def get_vcpu_used(self): + """This method is supported only libvirt. """ + return + + def get_memory_mb_used(self): + """This method is supported only libvirt. """ + return + + def get_local_gb_used(self): + """This method is supported only libvirt. """ return def get_hypervisor_type(self): -- cgit From d88d74c9a0a28e0ebd6cedf694753b9ee9decdac Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Fri, 18 Feb 2011 14:15:04 +0900 Subject: fixed based on reviewer's comment. 1. erase wrapper function(remove/exists/mktempfile) from nova.utils. 2. nova-manage service describeresource(->describe_resource) 3. nova-manage service updateresource(->update_resource) 4. erase "my mistake print" statement Additional changes are made at: 1. nova.image.s3.show 2. nova.compute.api.create that's because instances cannot launched without this changes. --- bin/nova-manage | 10 +++++----- nova/compute/api.py | 4 ++-- nova/compute/manager.py | 15 ++++++++++----- nova/db/sqlalchemy/migrate_repo/versions/002_bexar.py | 9 --------- nova/image/s3.py | 2 +- nova/scheduler/manager.py | 1 - nova/utils.py | 18 ------------------ nova/virt/disk.py | 1 - nova/virt/libvirt_conn.py | 2 +- 9 files changed, 19 insertions(+), 43 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 7336a582b..0bfe0d969 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -574,7 +574,7 @@ class ServiceCommands(object): return db.service_update(ctxt, svc['id'], {'disabled': True}) - def describeresource(self, host): + def describe_resource(self, host): """describe cpu/memory/hdd info for host.""" result = rpc.call(context.get_admin_context(), @@ -606,7 +606,7 @@ class ServiceCommands(object): val['memory_mb'], val['local_gb']) - def updateresource(self, host): + def update_resource(self, host): """update available vcpu/memory/disk info for host.""" ctxt = context.get_admin_context() @@ -618,9 +618,9 @@ class ServiceCommands(object): if len(service_refs) <= 0: raise exception.Invalid(_('%s is not compute node.') % host) - result = rpc.call(ctxt, - db.queue_get_for(ctxt, FLAGS.compute_topic, host), - {"method": "update_available_resource"}) + rpc.call(ctxt, + db.queue_get_for(ctxt, FLAGS.compute_topic, host), + {"method": "update_available_resource"}) class LogCommands(object): diff --git a/nova/compute/api.py b/nova/compute/api.py index ac02dbcfa..740dd3935 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -103,9 +103,9 @@ class API(base.Base): if not is_vpn: image = self.image_service.show(context, image_id) if kernel_id is None: - kernel_id = image.get('kernelId', None) + kernel_id = image.get('kernel_id', None) if ramdisk_id is None: - ramdisk_id = image.get('ramdiskId', None) + ramdisk_id = image.get('ramdisk_id', None) # No kernel and ramdisk for raw images if kernel_id == str(FLAGS.null_kernel): kernel_id = None diff --git a/nova/compute/manager.py b/nova/compute/manager.py index cae95dd93..47dd5fd5e 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -38,6 +38,8 @@ import datetime import random import string import socket +import os +import tempfile import time import functools @@ -577,14 +579,17 @@ class ComputeManager(manager.Manager): @exception.wrap_exception def mktmpfile(self, context): """make tmpfile under FLAGS.instance_path.""" - return utils.mktmpfile(FLAGS.instances_path) + fd, name = tempfile.mkstemp(dir=FLAGS.instances_path) + # No essential reason to write dateinfo. just for debugging reason. + os.fdopen(fd, 'w').write(str(datetime.datetime.utcnow())) + return name @exception.wrap_exception def confirm_tmpfile(self, context, path): """Confirm existence of the tmpfile given by path.""" - if not utils.exists(path): + if not os.path.exists(path): raise exception.NotFound(_('%s not found') % path) - return utils.remove(path) + return os.remove(path) @exception.wrap_exception def update_available_resource(self, context): @@ -683,7 +688,7 @@ class ComputeManager(manager.Manager): Post operations for live migration. Mainly, database updating. """ - LOG.info('post_live_migration() is started..') + LOG.info(_('post_live_migration() is started..')) instance_id = instance_ref['id'] # Detaching volumes. @@ -705,7 +710,7 @@ class ComputeManager(manager.Manager): # Not return if fixed_ip is not found, otherwise, # instance never be accessible.. if None == fixed_ip: - logging.warn('fixed_ip is not found for %s ' % i_name) + LOG.warn(_('fixed_ip is not found for %s.') % i_name) self.db.fixed_ip_update(ctxt, fixed_ip, {'host': dest}) try: diff --git a/nova/db/sqlalchemy/migrate_repo/versions/002_bexar.py b/nova/db/sqlalchemy/migrate_repo/versions/002_bexar.py index 38210db85..699b837f8 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/002_bexar.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/002_bexar.py @@ -229,12 +229,3 @@ def upgrade(migrate_engine): networks.create_column(networks_cidr_v6) networks.create_column(networks_ra_server) services.create_column(services_availability_zone) - #services.create_column(services_vcpus) - #services.create_column(services_memory_mb) - #services.create_column(services_local_gb) - #services.create_column(services_vcpus_used) - #services.create_column(services_memory_mb_used) - #services.create_column(services_local_gb_used) - #services.create_column(services_hypervisor_type) - #services.create_column(services_hypervisor_version) - #services.create_column(services_cpu_info) diff --git a/nova/image/s3.py b/nova/image/s3.py index 71304cdd6..14135a1ee 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -94,7 +94,7 @@ class S3ImageService(service.BaseImageService): if FLAGS.connection_type == 'fake': return {'imageId': 'bar'} result = self.index(context) - result = [i for i in result if i['imageId'] == image_id] + result = [i for i in result if i['id'] == image_id] if not result: raise exception.NotFound(_('Image %s could not be found') % image_id) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index af54c72be..ea7ae7bd2 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -60,7 +60,6 @@ class SchedulerManager(manager.Manager): host = getattr(self.driver, driver_method)(elevated, *args, **kwargs) except AttributeError, e: - print 'manager.attrerr', e host = self.driver.schedule(elevated, topic, *args, **kwargs) rpc.cast(context, diff --git a/nova/utils.py b/nova/utils.py index 966dde667..8d7ff1f64 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -337,24 +337,6 @@ def str_dict_replace(s, mapping): return s -def mktmpfile(dir): - """create tmpfile under dir, and return filename.""" - filename = datetime.datetime.utcnow().strftime('%Y%m%d%H%M%S') - fpath = os.path.join(dir, filename) - open(fpath, 'a+').write(fpath + '\n') - return fpath - - -def exists(filename): - """check file path existence.""" - return os.path.exists(filename) - - -def remove(filename): - """remove file.""" - return os.remove(filename) - - class LazyPluggable(object): """A pluggable backend loaded lazily based on some value.""" diff --git a/nova/virt/disk.py b/nova/virt/disk.py index ec4acc452..c5565abfa 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -112,7 +112,6 @@ def inject_data(image, key=None, net=None, partition=None, nbd=False): def _link_device(image, nbd): """Link image to device using loopback or nbd""" - print '_link_device:0:', nbd, '::', image if nbd: device = _allocate_device() utils.execute('sudo qemu-nbd -c %s %s' % (device, image)) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 9b7a9ddbe..579c4593e 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -977,7 +977,7 @@ class LibvirtConnection(object): """ Update compute manager resource info on Service table. This method is called when nova-coompute launches, and - whenever admin executes "nova-manage service updateresource". + whenever admin executes "nova-manage service update_resource". """ try: -- cgit From 205810c3da4652fd0f5203f53299cd998ac7cf82 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Fri, 18 Feb 2011 11:44:06 +0100 Subject: added functionality to list only fixed ip addresses of one node and added exception handling to list method # nova-manage fixed list XXXX network IP address MAC address hostname host 10.xx.xx.0/24 10.xx.xx.5 02:16:3e:3f:33:b6 i-00000547 XXXX 10.xx.xx.0/24 10.xx.xx.9 02:16:3e:14:03:d6 i-00000548 XXXX 10.xx.xx.0/24 10.xx.xx.12 02:16:3e:20:1b:e7 i-00000549 XXXX --- bin/nova-manage | 19 ++++++++++++------- nova/db/api.py | 5 +++++ nova/db/sqlalchemy/api.py | 22 ++++++++++++++++++++++ 3 files changed, 39 insertions(+), 7 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 6d67252b8..60a00f1cf 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -439,10 +439,15 @@ class FixedIpCommands(object): def list(self, host=None): """Lists all fixed ips (optionally by host) arguments: [host]""" ctxt = context.get_admin_context() - if host == None: - fixed_ips = db.fixed_ip_get_all(ctxt) - else: - fixed_ips = db.fixed_ip_get_all_by_host(ctxt, host) + + try: + if host == None: + fixed_ips = db.fixed_ip_get_all(ctxt) + else: + fixed_ips = db.fixed_ip_get_all_by_host(ctxt, host) + except exception.NotFound as ex: + print "error: %s" % ex + sys.exit(2) print "%-18s\t%-15s\t%-17s\t%-15s\t%s" % (_('network'), _('IP address'), @@ -459,9 +464,9 @@ class FixedIpCommands(object): host = instance['host'] mac_address = instance['mac_address'] print "%-18s\t%-15s\t%-17s\t%-15s\t%s" % ( - fixed_ip['network']['cidr'], - fixed_ip['address'], - mac_address, hostname, host) + fixed_ip['network']['cidr'], + fixed_ip['address'], + mac_address, hostname, host) class FloatingIpCommands(object): diff --git a/nova/db/api.py b/nova/db/api.py index d7f3746d2..6053c0352 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -293,6 +293,11 @@ def fixed_ip_get_all(context): return IMPL.fixed_ip_get_all(context) +def fixed_ip_get_all_by_host(context, host): + """Get all defined fixed ips used by a host.""" + return IMPL.fixed_ip_get_all_by_host(context, host) + + def fixed_ip_get_by_address(context, address): """Get a fixed ip by address or raise if it does not exist.""" return IMPL.fixed_ip_get_by_address(context, address) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 2697fac73..d07c53759 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -594,6 +594,28 @@ def fixed_ip_get_all(context, session=None): return result +@require_admin_context +def fixed_ip_get_all_by_host(context, host=None): + session = get_session() + + # FIXME: I'm sure that SQLAlchemy can handle this in a nicer way + instances = session.query(models.Instance).\ + filter_by(state=1).\ + filter_by(host=host).\ + all() + + result = [] + for instance in instances: + result.append(session.query(models.FixedIp).\ + filter_by(instance_id=instance['id']).\ + first()) + + if not result: + raise exception.NotFound(_('No fixed ips for this host defined')) + + return result + + @require_context def fixed_ip_get_by_address(context, address, session=None): if not session: -- cgit From cf8cf8287169e3e0b996db7db5a135dea88db63a Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Fri, 18 Feb 2011 12:01:50 +0100 Subject: added new class Instances to manage instances and added a new listing method into the class # nova-manage instance list instance node type state launched image kernel ramdisk project user zone index i-00000547 XXXXXXX m1.small running 2011-02-18 08:36:37 ami-a03ndz0q ami-0isqekvw testing berendt None 0 i-00000548 XXXXXXX m1.small running 2011-02-18 08:37:17 ami-a03ndz0q ami-0isqekvw testing berendt None 1 i-00000549 XXXXXXX m1.small running 2011-02-18 08:37:52 ami-a03ndz0q ami-0isqekvw testing berendt None 2 --- bin/nova-manage | 43 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 42 insertions(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index 60a00f1cf..68847bdde 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -618,6 +618,45 @@ class DbCommands(object): print migration.db_version() +class InstanceCommands(object): + """Class for managing instances.""" + + def list(self, host=None, instance=None): + """Show a list of all instances""" + print "%-10s %-15s %-10s %-10s %-19s %-12s %-12s" \ + " %-12s %-10s %-10s %-10s %-5s" % ( + _('instance'), + _('node'), + _('type'), + _('state'), + _('launched'), + _('image'), + _('kernel'), + _('ramdisk'), + _('project'), + _('user'), + _('zone'), + _('index'), + ) + + for instance in db.instance_get_all(context.get_admin_context()): + print "%-10s %-15s %-10s %-10s %-19s %-12s %-12s %-12s" \ + " %-10s %-10s %-10s %-5d" % ( + instance['hostname'], + instance['host'], + instance['instance_type'], + instance['state_description'], + instance['launched_at'], + instance['image_id'], + instance['kernel_id'], + instance['ramdisk_id'], + instance['project_id'], + instance['user_id'], + instance['availability_zone'], + instance['launch_index'] + ) + + class VolumeCommands(object): """Methods for dealing with a cloud in an odd state""" @@ -677,7 +716,9 @@ CATEGORIES = [ ('service', ServiceCommands), ('log', LogCommands), ('db', DbCommands), - ('volume', VolumeCommands)] + ('volume', VolumeCommands) + ('instance', InstanceCommands), +] def lazy_match(name, key_value_tuples): -- cgit From 764f0a457e74c4498cbc9ea30a184e61f7932072 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Tue, 22 Feb 2011 13:18:21 +0900 Subject: just add 005_add_live_migration.py. --- .../versions/005_add_live_migration.py | 84 ++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py diff --git a/nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py b/nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py new file mode 100644 index 000000000..903f7a646 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py @@ -0,0 +1,84 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +# +# New Tables +# + +compute_services = Table('compute_services', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('service_id', Integer(), nullable=False), + + Column('vcpus', Integer(), nullable=False), + Column('memory_mb', Integer(), nullable=False), + Column('local_gb', Integer(), nullable=False), + Column('vcpus_used', Integer(), nullable=False), + Column('memory_mb_used', Integer(), nullable=False), + Column('local_gb_used', Integer(), nullable=False), + Column('hypervisor_type', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=False), + Column('hypervisor_version', Integer(), nullable=False), + Column('cpu_info', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=False), + ) + + +# +# Tables to alter +# +instances_launched_on = Column( + 'launched_on', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=True) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + try: + compute_services.create() + except Exception: + logging.info(repr(compute_services)) + logging.exception('Exception while creating table') + meta.drop_all(tables=[compute_services]) + raise + + instances.create_column(instances_launched_on) -- cgit From c32e57999be09368b18f5a89315465e629ed4819 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Tue, 22 Feb 2011 23:55:03 +0900 Subject: Fixed based on reviewer's comment. 1. Change docstrings format 2. Fix comment grammer mistake, etc --- bin/nova-api | 2 - bin/nova-dhcpbridge | 1 - bin/nova-manage | 28 +- nova/compute/manager.py | 123 +++++--- nova/db/api.py | 12 +- nova/db/sqlalchemy/api.py | 27 +- .../versions/005_add_live_migration.py | 3 +- nova/db/sqlalchemy/models.py | 12 +- nova/scheduler/driver.py | 121 +++++--- nova/scheduler/manager.py | 15 +- nova/tests/test_compute.py | 77 ++--- nova/tests/test_scheduler.py | 141 ++++------ nova/tests/test_service.py | 6 +- nova/tests/test_virt.py | 309 +++++++++------------ nova/virt/fake.py | 74 +---- nova/virt/libvirt_conn.py | 182 ++++++++---- nova/virt/xenapi_conn.py | 14 +- nova/volume/driver.py | 14 +- 18 files changed, 576 insertions(+), 585 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index 59466a8c6..11176a021 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -38,13 +38,11 @@ from nova import flags from nova import log as logging from nova import version from nova import wsgi -from nova import utils logging.basicConfig() LOG = logging.getLogger('nova.api') LOG.setLevel(logging.DEBUG) -utils.default_flagfile() FLAGS = flags.FLAGS API_ENDPOINTS = ['ec2', 'osapi'] diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index fb04a484e..d38ba2543 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -125,7 +125,6 @@ def main(): LOG.debug(msg) globals()[action + '_lease'](mac, ip, hostname, interface) else: - open('/tmp/aaa', 'w+').write('-- %s' % interface) print init_leases(interface) if __name__ == "__main__": diff --git a/bin/nova-manage b/bin/nova-manage index 696ce0cad..49246fcc8 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -548,7 +548,12 @@ class InstanceCommands(object): """Class for mangaging VM instances.""" def live_migration(self, ec2_id, dest): - """Migrates a running instance to a new machine.""" + """Migrates a running instance to a new machine. + + :param ec2_id: instance id which comes from euca-describe-instance. + :param dest: destination host name. + + """ ctxt = context.get_admin_context() instance_id = ec2_id_to_id(ec2_id) @@ -569,9 +574,8 @@ class InstanceCommands(object): "dest": dest, "topic": FLAGS.compute_topic}}) - msg = 'Migration of %s initiated. ' % ec2_id - msg += 'Check its progress using euca-describe-instances.' - print msg + print _('Migration of %s initiated.' + 'Check its progress using euca-describe-instances.') % ec2_id class ServiceCommands(object): @@ -619,15 +623,17 @@ class ServiceCommands(object): db.service_update(ctxt, svc['id'], {'disabled': True}) def describe_resource(self, host): - """describe cpu/memory/hdd info for host.""" + """Describes cpu/memory/hdd info for host. + + :param host: hostname. + + """ result = rpc.call(context.get_admin_context(), FLAGS.scheduler_topic, - {"method": "show_host_resource", + {"method": "show_host_resources", "args": {"host": host}}) - # Checking result msg format is necessary, that will have done - # when this feture is included in API. if type(result) != dict: print 'Unexpected error occurs' print '[Result]', result @@ -650,7 +656,11 @@ class ServiceCommands(object): val['local_gb']) def update_resource(self, host): - """update available vcpu/memory/disk info for host.""" + """Updates available vcpu/memory/disk info for host. + + :param host: hostname. + + """ ctxt = context.get_admin_context() service_refs = db.service_get_all_by_host(ctxt, host) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index d548cef6f..5b6e9082e 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -36,10 +36,10 @@ terminating it. import base64 import datetime +import os import random import string import socket -import os import tempfile import time import functools @@ -65,8 +65,8 @@ flags.DEFINE_string('console_host', socket.gethostname(), 'Console proxy host to use to connect to instances on' 'this host.') flags.DEFINE_string('live_migration_retry_count', 30, - ("""Retry count needed in live_migration.""" - """ sleep 1 sec for each count""")) + ("Retry count needed in live_migration." + " sleep 1 sec for each count")) LOG = logging.getLogger('nova.compute.manager') @@ -602,31 +602,66 @@ class ComputeManager(manager.Manager): @exception.wrap_exception def compare_cpu(self, context, cpu_info): - """ Check the host cpu is compatible to a cpu given by xml.""" + """Checks the host cpu is compatible to a cpu given by xml. + + :param context: security context + :param cpu_info: json string obtained from virConnect.getCapabilities + :returns: See driver.compare_cpu + + """ return self.driver.compare_cpu(cpu_info) @exception.wrap_exception def mktmpfile(self, context): - """make tmpfile under FLAGS.instance_path.""" - fd, name = tempfile.mkstemp(dir=FLAGS.instances_path) - # No essential reason to write dateinfo. just for debugging reason. - os.fdopen(fd, 'w').write(str(datetime.datetime.utcnow())) + """Makes tmpfile under FLAGS.instance_path. + + This method enables compute nodes to recognize that they mounts + same shared storage. mktmpfile()/confirm_tmpfile is a pair. + + :param context: security context + :returns: tmpfile name + + """ + + dirpath = FLAGS.instances_path + fd, name = tempfile.mkstemp(dir=dirpath) + LOG.debug(_("Creating tmpfile %s to notify to other " + "compute node that they mounts same storage.") % name) + os.fdopen(fd, 'w+').close() return name @exception.wrap_exception def confirm_tmpfile(self, context, path): - """Confirm existence of the tmpfile given by path.""" + """Confirms existence of the tmpfile given by path. + + :param context: security context + :param path: confirm existence of this path + :returns: depends on os.remove() + + """ + if not os.path.exists(path): raise exception.NotFound(_('%s not found') % path) return os.remove(path) @exception.wrap_exception def update_available_resource(self, context): - """See comments update_resource_info""" + """See comments update_resource_info. + + :param context: security context + :returns: See driver.update_available_resource() + + """ + return self.driver.update_available_resource(context, self.host) def pre_live_migration(self, context, instance_id): - """Any preparation for live migration at dst host.""" + """Preparations for live migration at dest host. + + :param context: security context + :param instance_id: nova.db.sqlalchemy.models.Instance.Id + + """ # Getting instance info instance_ref = self.db.instance_get(context, instance_id) @@ -635,7 +670,7 @@ class ComputeManager(manager.Manager): # Getting fixed ips fixed_ip = self.db.instance_get_fixed_address(context, instance_id) if not fixed_ip: - msg = _("%(instance_id)s(%(ec2_id)s) doesnt have fixed_ip") + msg = _("%(instance_id)s(%(ec2_id)s) does'nt have fixed_ip") raise exception.NotFound(msg % locals()) # If any volume is mounted, prepare here. @@ -645,8 +680,8 @@ class ComputeManager(manager.Manager): for v in instance_ref['volumes']: self.volume_manager.setup_compute_volume(context, v['id']) - # Bridge settings - # call this method prior to ensure_filtering_rules_for_instance, + # Bridge settings. + # Call this method prior to ensure_filtering_rules_for_instance, # since bridge is not set up, ensure_filtering_rules_for instance # fails. # @@ -660,24 +695,29 @@ class ComputeManager(manager.Manager): break except exception.ProcessExecutionError, e: if i == max_retry - 1: - raise e + raise else: - LOG.warn(_("setup_compute_network() fail %(i)d th. " - "Retry up to %(max_retry)d for %(ec2_id)s") + LOG.warn(_("setup_compute_network() failed %(i)d." + "Retry up to %(max_retry)d for %(ec2_id)s.") % locals()) time.sleep(1) # Creating filters to hypervisors and firewalls. # An example is that nova-instance-instance-xxx, - # which is written to libvirt.xml( check "virsh nwfilter-list ) - # On destination host, this nwfilter is necessary. + # which is written to libvirt.xml(Check "virsh nwfilter-list") + # This nwfilter is necessary on the destination host. # In addition, this method is creating filtering rule # onto destination host. self.driver.ensure_filtering_rules_for_instance(instance_ref) - #@exception.wrap_exception def live_migration(self, context, instance_id, dest): - """Executing live migration.""" + """Executing live migration. + + :param context: security context + :param instance_id: nova.db.sqlalchemy.models.Instance.Id + :param dest: destination host + + """ # Get instance for error handling. instance_ref = self.db.instance_get(context, instance_id) @@ -702,7 +742,7 @@ class ComputeManager(manager.Manager): msg = _("Pre live migration for %(i_name)s failed at %(dest)s") LOG.error(msg % locals()) self.recover_live_migration(context, instance_ref) - raise e + raise # Executing live migration # live_migration might raises exceptions, but @@ -712,10 +752,17 @@ class ComputeManager(manager.Manager): self.recover_live_migration) def post_live_migration(self, ctxt, instance_ref, dest): + """Post operations for live migration. + + This method is called from live_migration + and mainly updating database record. + + :param ctxt: security context + :param instance_id: nova.db.sqlalchemy.models.Instance.Id + :param dest: destination host + """ - Post operations for live migration. - Mainly, database updating. - """ + LOG.info(_('post_live_migration() is started..')) instance_id = instance_ref['id'] @@ -734,19 +781,12 @@ class ComputeManager(manager.Manager): # Database updating. i_name = instance_ref.name - #fixed_ip = self.db.instance_get_fixed_address(ctxt, instance_id) - # Not return if fixed_ip is not found, otherwise, - # instance never be accessible.. - #if None == fixed_ip: - # LOG.warn(_('fixed_ip is not found for %s.') % i_name) - #self.db.fixed_ip_update(ctxt, fixed_ip, {'host': dest}) - try: # Not return if floating_ip is not found, otherwise, # instance never be accessible.. floating_ip = self.db.instance_get_floating_address(ctxt, instance_id) - if None == floating_ip: + if not floating_ip: LOG.info(_('floating_ip is not found for %s'), i_name) else: floating_ip_ref = self.db.floating_ip_get_by_address(ctxt, @@ -763,15 +803,23 @@ class ComputeManager(manager.Manager): # Restore instance/volume state self.recover_live_migration(ctxt, instance_ref, dest) - msg = _('Migrating %(i_name)s to %(dest)s finishes successfully.') - LOG.info(msg % locals()) + LOG.info(_('Migrating %(i_name)s to %(dest)s finishes successfully.') + % locals()) LOG.info(_("The below error is normally occurs." "Just check if instance is successfully migrated.\n" "libvir: QEMU error : Domain not found: no domain " "with matching name..")) def recover_live_migration(self, ctxt, instance_ref, host=None): - """Instance/volume state is recovered from migrating -> running.""" + """Recovers Instance/volume state from migrating -> running. + + :param ctxt: security context + :param instance_id: nova.db.sqlalchemy.models.Instance.Id + :param host: + DB column value is updated by this hostname. + if none, the host instance currently running is selected. + + """ if not host: host = instance_ref['host'] @@ -783,5 +831,4 @@ class ComputeManager(manager.Manager): 'host': host}) for v in instance_ref['volumes']: - self.db.volume_update(ctxt, v['id'], {'status': 'in-use', - 'host': host}) + self.db.volume_update(ctxt, v['id'], {'status': 'in-use'}) diff --git a/nova/db/api.py b/nova/db/api.py index 609f62495..e10a06178 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -169,6 +169,7 @@ def compute_service_update(context, compute_id, values): Raises NotFound if computeService does not exist. """ + return IMPL.compute_service_update(context, compute_id, values) @@ -446,27 +447,22 @@ def instance_add_security_group(context, instance_id, security_group_id): security_group_id) -def instance_get_all_by_host(context, hostname): - """Get instances by host""" - return IMPL.instance_get_all_by_host(context, hostname) - - def instance_get_vcpu_sum_by_host_and_project(context, hostname, proj_id): - """Get instances.vcpus by host and project""" + """Get instances.vcpus by host and project.""" return IMPL.instance_get_vcpu_sum_by_host_and_project(context, hostname, proj_id) def instance_get_memory_sum_by_host_and_project(context, hostname, proj_id): - """Get amount of memory by host and project """ + """Get amount of memory by host and project.""" return IMPL.instance_get_memory_sum_by_host_and_project(context, hostname, proj_id) def instance_get_disk_sum_by_host_and_project(context, hostname, proj_id): - """Get total amount of disk by host and project """ + """Get total amount of disk by host and project.""" return IMPL.instance_get_disk_sum_by_host_and_project(context, hostname, proj_id) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 43d56cd8a..b4f45a089 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -184,8 +184,8 @@ def service_get_all_compute_by_host(context, host): all() if not result: - msg = _('%s does not exist or not compute node') - raise exception.NotFound(msg % host) + raise exception.NotFound(_("%s does not exist or not " + "compute node.") % host) return result @@ -328,7 +328,7 @@ def compute_service_create(context, values): def compute_service_update(context, compute_id, values): session = get_session() with session.begin(): - compute_ref = service_get(context, compute_id, session=session) + compute_ref = compute_service_get(context, compute_id, session=session) compute_ref.update(values) compute_ref.save(session=session) @@ -964,21 +964,6 @@ def instance_add_security_group(context, instance_id, security_group_id): instance_ref.save(session=session) -@require_context -def instance_get_all_by_host(context, hostname): - session = get_session() - if not session: - session = get_session() - - result = session.query(models.Instance).\ - filter_by(host=hostname).\ - filter_by(deleted=can_read_deleted(context)).\ - all() - if not result: - return [] - return result - - @require_context def instance_get_vcpu_sum_by_host_and_project(context, hostname, proj_id): session = get_session() @@ -987,7 +972,7 @@ def instance_get_vcpu_sum_by_host_and_project(context, hostname, proj_id): filter_by(project_id=proj_id).\ filter_by(deleted=False).\ value(func.sum(models.Instance.vcpus)) - if None == result: + if not result: return 0 return result @@ -1000,7 +985,7 @@ def instance_get_memory_sum_by_host_and_project(context, hostname, proj_id): filter_by(project_id=proj_id).\ filter_by(deleted=False).\ value(func.sum(models.Instance.memory_mb)) - if None == result: + if not result: return 0 return result @@ -1013,7 +998,7 @@ def instance_get_disk_sum_by_host_and_project(context, hostname, proj_id): filter_by(project_id=proj_id).\ filter_by(deleted=False).\ value(func.sum(models.Instance.local_gb)) - if None == result: + if not result: return 0 return result diff --git a/nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py b/nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py index 903f7a646..2689b5b74 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/005_add_live_migration.py @@ -16,10 +16,9 @@ # License for the specific language governing permissions and limitations # under the License. -from sqlalchemy import * from migrate import * - from nova import log as logging +from sqlalchemy import * meta = MetaData() diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 02d4e2f9b..f2a029c20 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -137,12 +137,14 @@ class ComputeService(BASE, NovaBase): # Note(masumotok): Expected Strings example: # - # '{"arch":"x86_64", "model":"Nehalem", - # "topology":{"sockets":1, "threads":2, "cores":3}, - # features:[ "tdtscp", "xtpr"]}' + # '{"arch":"x86_64", + # "model":"Nehalem", + # "topology":{"sockets":1, "threads":2, "cores":3}, + # "features":["tdtscp", "xtpr"]}' # # Points are "json translatable" and it must have all dictionary keys - # above, and tag of getCapabilities()(See libvirt.virtConnection). + # above, since it is copied from tag of getCapabilities() + # (See libvirt.virtConnection). cpu_info = Column(Text, nullable=True) @@ -220,7 +222,7 @@ class Instance(BASE, NovaBase): display_description = Column(String(255)) # To remember on which host a instance booted. - # An instance may moved to other host by live migraiton. + # An instance may have moved to another host by live migraiton. launched_on = Column(Text) locked = Column(Boolean) diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 41b87bbca..8c30702ba 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -70,9 +70,18 @@ class Scheduler(object): raise NotImplementedError(_("Must implement a fallback schedule")) def schedule_live_migration(self, context, instance_id, dest): - """live migration method""" + """Live migration scheduling method. - # Whether instance exists and running + :param context: + :param instance_id: + :param dest: destination host + :return: + The host where instance is running currently. + Then scheduler send request that host. + + """ + + # Whether instance exists and is running. instance_ref = db.instance_get(context, instance_id) # Checking instance. @@ -102,11 +111,16 @@ class Scheduler(object): return src def _live_migration_src_check(self, context, instance_ref): - """Live migration check routine (for src host)""" + """Live migration check routine (for src host). + + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + + """ # Checking instance is running. - if power_state.RUNNING != instance_ref['state'] or \ - 'running' != instance_ref['state_description']: + if (power_state.RUNNING != instance_ref['state'] or \ + 'running' != instance_ref['state_description']): msg = _('Instance(%s) is not running') ec2_id = instance_ref['hostname'] raise exception.Invalid(msg % ec2_id) @@ -129,7 +143,13 @@ class Scheduler(object): raise exception.Invalid(msg % src) def _live_migration_dest_check(self, context, instance_ref, dest): - """Live migration check routine (for destination host)""" + """Live migration check routine (for destination host). + + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + :param dest: destination host + + """ # Checking dest exists and compute node. dservice_refs = db.service_get_all_compute_by_host(context, dest) @@ -145,20 +165,25 @@ class Scheduler(object): src = instance_ref['host'] if dest == src: ec2_id = instance_ref['hostname'] - msg = _("""%(dest)s is where %(ec2_id)s is """ - """running now. choose other host.""") % locals() - raise exception.Invalid(msg) + raise exception.Invalid(_("%(dest)s is where %(ec2_id)s is " + "running now. choose other host.") + % locals()) # Checking dst host still has enough capacities. - self.has_enough_resource(context, instance_ref, dest) + self.has_enough_resources(context, instance_ref, dest) def _live_migration_common_check(self, context, instance_ref, dest): - """ - Live migration check routine. - Below pre-checkings are followed by + """Live migration common check routine. + + Below checkings are followed by http://wiki.libvirt.org/page/TodoPreMigrationChecks + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + :param dest: destination host + """ + # Checking shared storage connectivity self.mounted_on_same_shared_storage(context, instance_ref, dest) @@ -168,27 +193,27 @@ class Scheduler(object): # Checking original host( where instance was launched at) exists. try: - oservice_refs = \ - db.service_get_all_compute_by_host(context, - instance_ref['launched_on']) + oservice_refs = db.service_get_all_compute_by_host(context, + instance_ref['launched_on']) except exception.NotFound: - msg = _('%s(where instance was launched at) does not exists.') - raise exception.Invalid(msg % instance_ref['launched_on']) + raise exception.Invalid(_("host %s where instance was launched " + "does not exist.") + % instance_ref['launched_on']) oservice_ref = oservice_refs[0]['compute_service'][0] # Checking hypervisor is same. o = oservice_ref['hypervisor_type'] d = dservice_ref['hypervisor_type'] if o != d: - msg = _('Different hypervisor type(%(o)s->%(d)s)') % locals() - raise exception.Invalid(msg) + raise exception.Invalid(_("Different hypervisor type" + "(%(o)s->%(d)s)')" % locals())) # Checkng hypervisor version. o = oservice_ref['hypervisor_version'] d = dservice_ref['hypervisor_version'] if o > d: - msg = _('Older hypervisor version(%(o)s->%(d)s)') % locals() - raise exception.Invalid(msg) + raise exception.Invalid(_('Older hypervisor version(%(o)s->%(d)s)') + % locals()) # Checking cpuinfo. try: @@ -200,19 +225,24 @@ class Scheduler(object): except rpc.RemoteError, e: ec2_id = instance_ref['hostname'] src = instance_ref['host'] - msg = _("""%(dest)s doesnt have compatibility to %(src)s""" - """(where %(ec2_id)s was launched at)""") - logging.exception(msg % locals()) - raise e + logging.exception(_("host %(dest)s is not compatible with " + "original host %(src)s.") % locals()) + raise + + def has_enough_resources(self, context, instance_ref, dest): + """Checks if destination host has enough resource for live migration. - def has_enough_resource(self, context, instance_ref, dest): - """ - Check if destination host has enough resource for live migration. Currently, only memory checking has been done. If storage migration(block migration, meaning live-migration without any shared storage) will be available, local storage checking is also necessary. + + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + :param dest: destination host + """ + # Getting instance information ec2_id = instance_ref['hostname'] @@ -225,15 +255,23 @@ class Scheduler(object): mem_avail = mem_total - mem_used mem_inst = instance_ref['memory_mb'] if mem_avail <= mem_inst: - msg = _("""%(ec2_id)s is not capable to migrate %(dest)s""" - """(host:%(mem_avail)s <= instance:%(mem_inst)s)""") - raise exception.NotEmpty(msg % locals()) + raise exception.NotEmpty(_("%(ec2_id)s is not capable to " + "migrate %(dest)s (host:%(mem_avail)s " + " <= instance:%(mem_inst)s)") + % locals()) def mounted_on_same_shared_storage(self, context, instance_ref, dest): + """Check if the src and dest host mount same shared storage. + + At first, dest host creates temp file, and src host can see + it if they mounts same shared storage. Then src host erase it. + + :param context: security context + :param instance_ref: nova.db.sqlalchemy.models.Instance object + :param dest: destination host + """ - Check if /nova-inst-dir/insntances is mounted same storage at - live-migration src and dest host. - """ + src = instance_ref['host'] dst_t = db.queue_get_for(context, FLAGS.compute_topic, dest) src_t = db.queue_get_for(context, FLAGS.compute_topic, src) @@ -243,8 +281,8 @@ class Scheduler(object): filename = rpc.call(context, dst_t, {"method": 'mktmpfile'}) except rpc.RemoteError, e: msg = _("Cannot create tmpfile at %s to confirm shared storage.") - logging.error(msg % FLAGS.instance_path) - raise e + LOG.error(msg % FLAGS.instances_path) + raise # make sure existence at src host. try: @@ -252,8 +290,7 @@ class Scheduler(object): {"method": 'confirm_tmpfile', "args": {'path': filename}}) except (rpc.RemoteError, exception.NotFound), e: - ipath = FLAGS.instance_path - msg = _("""Cannot comfirm %(ipath)s at %(dest)s is located at""" - """ same shared storage.""") % locals() - logging.error(msg) - raise e + ipath = FLAGS.instances_path + logging.error(_("Cannot comfirm %(ipath)s at %(dest)s is " + "located at same shared storage.") % locals()) + raise diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 584dc49d2..783594c6f 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -69,10 +69,19 @@ class SchedulerManager(manager.Manager): LOG.debug(_("Casting to %(topic)s %(host)s for %(method)s") % locals()) # NOTE (masumotok) : This method should be moved to nova.api.ec2.admin. - # Based on bear design summit discussion, + # Based on bexar design summit discussion, # just put this here for bexar release. - def show_host_resource(self, context, host, *args): - """show the physical/usage resource given by hosts.""" + def show_host_resources(self, context, host, *args): + """Shows the physical/usage resource given by hosts. + + :param context: security context + :param host: hostname + :returns: + example format is below. + {'resource':D, 'usage':{proj_id1:D, proj_id2:D}} + D: {'vcpus':3, 'memory_mb':2048, 'local_gb':2048} + + """ compute_ref = db.service_get_all_compute_by_host(context, host) compute_ref = compute_ref[0] diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 74cb82eeb..3c88d186d 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -314,10 +314,7 @@ class ComputeTestCase(test.TestCase): self.compute_driver = utils.import_object(FLAGS.compute_driver) def test_pre_live_migration_instance_has_no_fixed_ip(self): - """ - if instances that are intended to be migrated doesnt have fixed_ip - (not happens usually), pre_live_migration has to raise Exception. - """ + """Confirm raising exception if instance doesn't have fixed_ip.""" instance_ref = self._get_dummy_instance() c = context.get_admin_context() i_id = instance_ref['id'] @@ -331,14 +328,9 @@ class ComputeTestCase(test.TestCase): self.assertRaises(exception.NotFound, self.compute.pre_live_migration, c, instance_ref['id']) - self.mox.ResetAll() def test_pre_live_migration_instance_has_volume(self): - """if any volumes are attached to the instances that are - intended to be migrated, setup_compute_volume must be - called because aoe module should be inserted at destination - host. This testcase checks on it. - """ + """Confirm setup_compute_volume is called when volume is mounted.""" i_ref = self._get_dummy_instance() c = context.get_admin_context() @@ -364,14 +356,9 @@ class ComputeTestCase(test.TestCase): self.mox.ReplayAll() ret = self.compute.pre_live_migration(c, i_ref['id']) self.assertEqual(ret, None) - self.mox.ResetAll() def test_pre_live_migration_instance_has_no_volume(self): - """if any volumes are not attached to the instances that are - intended to be migrated, log message should be appears - because administrator can proove instance conditions before - live_migration if any trouble occurs. - """ + """Confirm log meg when instance doesn't mount any volumes.""" i_ref = self._get_dummy_instance() i_ref.__setitem__('volumes', []) c = context.get_admin_context() @@ -395,14 +382,14 @@ class ComputeTestCase(test.TestCase): self.mox.ReplayAll() ret = self.compute.pre_live_migration(c, i_ref['id']) self.assertEqual(ret, None) - self.mox.ResetAll() def test_pre_live_migration_setup_compute_node_fail(self): - """setup_compute_node sometimes fail since concurrent request - comes to iptables and iptables complains. Then this method - tries to retry, but raise exception in case of over - max_retry_count. this method confirms raising exception. + """Confirm operation setup_compute_network() fails. + + It retries and raise exception when timeout exceeded. + """ + i_ref = self._get_dummy_instance() c = context.get_admin_context() @@ -427,14 +414,9 @@ class ComputeTestCase(test.TestCase): self.assertRaises(exception.ProcessExecutionError, self.compute.pre_live_migration, c, i_ref['id']) - self.mox.ResetAll() - def test_live_migration_instance_has_volume(self): - """Any volumes are mounted by instances to be migrated are found, - vblade health must be checked before starting live-migration. - And that is checked by check_for_export(). - This testcase confirms check_for_export() is called. - """ + def test_live_migration_works_correctly_with_volume(self): + """Confirm check_for_export to confirm volume health check.""" i_ref = self._get_dummy_instance() c = context.get_admin_context() topic = db.queue_get_for(c, FLAGS.compute_topic, i_ref['host']) @@ -457,15 +439,9 @@ class ComputeTestCase(test.TestCase): self.mox.ReplayAll() ret = self.compute.live_migration(c, i_ref['id'], i_ref['host']) self.assertEqual(ret, None) - self.mox.ResetAll() - - def test_live_migration_instance_has_volume_and_exception(self): - """In addition to test_live_migration_instance_has_volume testcase, - this testcase confirms if any exception raises from - check_for_export(). Then, valid seaquence of this method should - recovering instance/volumes status(ex. instance['state_description'] - is changed from 'migrating' -> 'running', was changed by scheduler) - """ + + def test_live_migration_dest_raises_exception(self): + """Confirm exception when pre_live_migration fails.""" i_ref = self._get_dummy_instance() c = context.get_admin_context() topic = db.queue_get_for(c, FLAGS.compute_topic, i_ref['host']) @@ -484,20 +460,16 @@ class ComputeTestCase(test.TestCase): 'state': power_state.RUNNING, 'host': i_ref['host']}) for v in i_ref['volumes']: - dbmock.volume_update(c, v['id'], {'status': 'in-use', - 'host': i_ref['host']}) + dbmock.volume_update(c, v['id'], {'status': 'in-use'}) self.compute.db = dbmock self.mox.ReplayAll() self.assertRaises(rpc.RemoteError, self.compute.live_migration, c, i_ref['id'], i_ref['host']) - self.mox.ResetAll() - def test_live_migration_instance_has_no_volume_and_exception(self): - """Simpler than - test_live_migration_instance_has_volume_and_exception - """ + def test_live_migration_dest_raises_exception_no_volume(self): + """Same as above test(input pattern is different) """ i_ref = self._get_dummy_instance() i_ref.__setitem__('volumes', []) c = context.get_admin_context() @@ -520,10 +492,9 @@ class ComputeTestCase(test.TestCase): self.assertRaises(rpc.RemoteError, self.compute.live_migration, c, i_ref['id'], i_ref['host']) - self.mox.ResetAll() - def test_live_migration_instance_has_no_volume(self): - """Simpler than test_live_migration_instance_has_volume.""" + def test_live_migration_works_correctly_no_volume(self): + """Confirm live_migration() works as expected correctly.""" i_ref = self._get_dummy_instance() i_ref.__setitem__('volumes', []) c = context.get_admin_context() @@ -545,11 +516,9 @@ class ComputeTestCase(test.TestCase): self.mox.ReplayAll() ret = self.compute.live_migration(c, i_ref['id'], i_ref['host']) self.assertEqual(ret, None) - self.mox.ResetAll() def test_post_live_migration_working_correctly(self): - """post_live_migration works as expected correctly """ - + """Confirm post_live_migration() works as expected correctly.""" dest = 'desthost' flo_addr = '1.2.1.2' @@ -579,19 +548,15 @@ class ComputeTestCase(test.TestCase): # executing self.mox.ReplayAll() ret = self.compute.post_live_migration(c, i_ref, dest) - self.mox.UnsetStubs() # make sure every data is rewritten to dest i_ref = db.instance_get(c, i_ref['id']) c1 = (i_ref['host'] == dest) - v_ref = db.volume_get(c, v_ref['id']) - c2 = (v_ref['host'] == dest) - c3 = False flo_refs = db.floating_ip_get_all_by_host(c, dest) - c3 = (len(flo_refs) != 0 and flo_refs[0]['address'] == flo_addr) + c2 = (len(flo_refs) != 0 and flo_refs[0]['address'] == flo_addr) # post operaton - self.assertTrue(c1 and c2 and c3) + self.assertTrue(c1 and c2) db.instance_destroy(c, instance_id) db.volume_destroy(c, v_ref['id']) db.floating_ip_destroy(c, flo_addr) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 729bcb580..301106848 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -108,22 +108,21 @@ class SchedulerTestCase(test.TestCase): self.mox.ReplayAll() scheduler.named_method(ctxt, 'topic', num=7) - def test_show_host_resource_host_not_exit(self): - """ - A testcase of driver.has_enough_resource - given host does not exists. - """ + def test_show_host_resources_host_not_exit(self): + """A host given as an argument does not exists.""" + scheduler = manager.SchedulerManager() dest = 'dummydest' ctxt = context.get_admin_context() try: - scheduler.show_host_resource(ctxt, dest) + scheduler.show_host_resources(ctxt, dest) except exception.NotFound, e: c1 = (0 <= e.message.find('does not exist or not compute node')) self.assertTrue(c1) def _dic_is_equal(self, dic1, dic2, keys=None): + """Compares 2 dictionary contents(Helper method)""" if not keys: keys = ['vcpus', 'memory_mb', 'local_gb', 'vcpus_used', 'memory_mb_used', 'local_gb_used'] @@ -133,16 +132,14 @@ class SchedulerTestCase(test.TestCase): return False return True - def test_show_host_resource_no_project(self): - """ - A testcase of driver.show_host_resource - no instance stays on the given host - """ + def test_show_host_resources_no_project(self): + """No instance are running on the given host.""" + scheduler = manager.SchedulerManager() ctxt = context.get_admin_context() s_ref = self._create_compute_service() - result = scheduler.show_host_resource(ctxt, s_ref['host']) + result = scheduler.show_host_resources(ctxt, s_ref['host']) # result checking c1 = ('resource' in result and 'usage' in result) @@ -152,11 +149,9 @@ class SchedulerTestCase(test.TestCase): self.assertTrue(c1 and c2 and c3) db.service_destroy(ctxt, s_ref['id']) - def test_show_host_resource_works_correctly(self): - """ - A testcase of driver.show_host_resource - to make sure everything finished with no error. - """ + def test_show_host_resources_works_correctly(self): + """show_host_resources() works correctly as expected.""" + scheduler = manager.SchedulerManager() ctxt = context.get_admin_context() s_ref = self._create_compute_service() @@ -164,7 +159,7 @@ class SchedulerTestCase(test.TestCase): i_ref2 = self._create_instance(project_id='p-02', vcpus=3, host=s_ref['host']) - result = scheduler.show_host_resource(ctxt, s_ref['host']) + result = scheduler.show_host_resources(ctxt, s_ref['host']) c1 = ('resource' in result and 'usage' in result) compute_service = s_ref['compute_service'][0] @@ -284,6 +279,7 @@ class SimpleDriverTestCase(test.TestCase): return db.volume_create(self.context, vol)['id'] def _create_compute_service(self, **kwargs): + """Create a compute service.""" dic = {'binary': 'nova-compute', 'topic': 'compute', 'report_count': 0, 'availability_zone': 'dummyzone'} @@ -698,13 +694,13 @@ class SimpleDriverTestCase(test.TestCase): volume1.kill() volume2.kill() - def test_scheduler_live_migraiton_with_volume(self): - """ - driver.scheduler_live_migration finishes successfully - (volumes are attached to instances) - This testcase make sure schedule_live_migration - changes instance state from 'running' -> 'migrating' + def test_scheduler_live_migration_with_volume(self): + """scheduler_live_migration() works correctly as expected. + + Also, checks instance state is changed from 'running' -> 'migrating'. + """ + instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) dic = {'instance_id': instance_id, 'size': 1} @@ -737,11 +733,9 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.volume_destroy(self.context, v_ref['id']) - def test_live_migraiton_src_check_instance_not_running(self): - """ - A testcase of driver._live_migration_src_check. - The instance given by instance_id is not running. - """ + def test_live_migration_src_check_instance_not_running(self): + """The instance given by instance_id is not running.""" + instance_id = self._create_instance(state_description='migrating') i_ref = db.instance_get(self.context, instance_id) @@ -754,12 +748,9 @@ class SimpleDriverTestCase(test.TestCase): self.assertTrue(c) db.instance_destroy(self.context, instance_id) - def test_live_migraiton_src_check_volume_node_not_alive(self): - """ - A testcase of driver._live_migration_src_check. - Volume node is not alive if any volumes are attached to - the given instance. - """ + def test_live_migration_src_check_volume_node_not_alive(self): + """Raise exception when volume node is not alive.""" + instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) dic = {'instance_id': instance_id, 'size': 1} @@ -782,11 +773,8 @@ class SimpleDriverTestCase(test.TestCase): db.service_destroy(self.context, s_ref['id']) db.volume_destroy(self.context, v_ref['id']) - def test_live_migraiton_src_check_compute_node_not_alive(self): - """ - A testcase of driver._live_migration_src_check. - The testcase make sure src-compute node is alive. - """ + def test_live_migration_src_check_compute_node_not_alive(self): + """Confirms src-compute node is alive.""" instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) t = datetime.datetime.utcnow() - datetime.timedelta(10) @@ -803,11 +791,8 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) - def test_live_migraiton_src_check_works_correctly(self): - """ - A testcase of driver._live_migration_src_check. - The testcase make sure everything finished with no error. - """ + def test_live_migration_src_check_works_correctly(self): + """Confirms this method finishes with no error.""" instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) s_ref = self._create_compute_service(host=i_ref['host']) @@ -819,11 +804,8 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) - def test_live_migraiton_dest_check_not_alive(self): - """ - A testcase of driver._live_migration_dst_check. - Destination host does not exist. - """ + def test_live_migration_dest_check_not_alive(self): + """Confirms exception raises in case dest host does not exist.""" instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) t = datetime.datetime.utcnow() - datetime.timedelta(10) @@ -841,11 +823,8 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) - def test_live_migraiton_dest_check_service_same_host(self): - """ - A testcase of driver._live_migration_dst_check. - Destination host is same as src host. - """ + def test_live_migration_dest_check_service_same_host(self): + """Confirms exceptioin raises in case dest and src is same host.""" instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) s_ref = self._create_compute_service(host=i_ref['host']) @@ -861,11 +840,8 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) - def test_live_migraiton_dest_check_service_lack_memory(self): - """ - A testcase of driver._live_migration_dst_check. - destination host doesnt have enough memory. - """ + def test_live_migration_dest_check_service_lack_memory(self): + """Confirms exception raises when dest doesn't have enough memory.""" instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) s_ref = self._create_compute_service(host='somewhere', @@ -882,11 +858,8 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) - def test_live_migraiton_dest_check_service_works_correctly(self): - """ - A testcase of driver._live_migration_dst_check. - The testcase make sure everything finished with no error. - """ + def test_live_migration_dest_check_service_works_correctly(self): + """Confirms method finishes with no error.""" instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) s_ref = self._create_compute_service(host='somewhere', @@ -899,13 +872,11 @@ class SimpleDriverTestCase(test.TestCase): db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) - def test_live_migraiton_common_check_service_orig_not_exists(self): - """ - A testcase of driver._live_migration_common_check. - Destination host does not exist. - """ + def test_live_migration_common_check_service_orig_not_exists(self): + """Destination host does not exist.""" + dest = 'dummydest' - # mocks for live_migraiton_common_check() + # mocks for live_migration_common_check() instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) t1 = datetime.datetime.utcnow() - datetime.timedelta(10) @@ -929,18 +900,15 @@ class SimpleDriverTestCase(test.TestCase): i_ref, dest) except exception.Invalid, e: - c = (e.message.find('does not exists') >= 0) + c = (e.message.find('does not exist') >= 0) self.assertTrue(c) self.mox.UnsetStubs() db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) - def test_live_migraiton_common_check_service_different_hypervisor(self): - """ - A testcase of driver._live_migration_common_check. - Original host and dest host has different hypervisor type. - """ + def test_live_migration_common_check_service_different_hypervisor(self): + """Original host and dest host has different hypervisor type.""" dest = 'dummydest' instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) @@ -969,11 +937,8 @@ class SimpleDriverTestCase(test.TestCase): db.service_destroy(self.context, s_ref['id']) db.service_destroy(self.context, s_ref2['id']) - def test_live_migraiton_common_check_service_different_version(self): - """ - A testcase of driver._live_migration_common_check. - Original host and dest host has different hypervisor version. - """ + def test_live_migration_common_check_service_different_version(self): + """Original host and dest host has different hypervisor version.""" dest = 'dummydest' instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) @@ -1003,11 +968,9 @@ class SimpleDriverTestCase(test.TestCase): db.service_destroy(self.context, s_ref['id']) db.service_destroy(self.context, s_ref2['id']) - def test_live_migraiton_common_check_service_checking_cpuinfo_fail(self): - """ - A testcase of driver._live_migration_common_check. - Original host and dest host has different hypervisor version. - """ + def test_live_migration_common_check_checking_cpuinfo_fail(self): + """Raise excetion when original host doen't have compatible cpu.""" + dest = 'dummydest' instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) @@ -1025,7 +988,7 @@ class SimpleDriverTestCase(test.TestCase): rpc.call(mox.IgnoreArg(), mox.IgnoreArg(), {"method": 'compare_cpu', "args": {'cpu_info': s_ref2['compute_service'][0]['cpu_info']}}).\ - AndRaise(rpc.RemoteError('doesnt have compatibility to', '', '')) + AndRaise(rpc.RemoteError("doesn't have compatibility to", "", "")) self.mox.ReplayAll() try: @@ -1033,7 +996,7 @@ class SimpleDriverTestCase(test.TestCase): i_ref, dest) except rpc.RemoteError, e: - c = (e.message.find(_('doesnt have compatibility to')) >= 0) + c = (e.message.find(_("doesn't have compatibility to")) >= 0) self.assertTrue(c) self.mox.UnsetStubs() diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index cb65584cf..bbd5c6d92 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -279,11 +279,7 @@ class ServiceTestCase(test.TestCase): self.assert_(not serv.model_disconnected) def test_compute_can_update_available_resource(self): - """ - Test nova-compute successfully updated Service table on DB. - Doing so, self.manager.update_service must be called - if 'self.binary == nova-compute', and this testcase checks on it. - """ + """Confirm compute updates their record of compute-service table.""" host = 'foo' binary = 'nova-compute' topic = 'compute1' diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 8ed726c21..91bdfcc5a 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -23,8 +23,8 @@ from nova import context from nova import db from nova import exception from nova import flags -from nova import test from nova import logging +from nova import test from nova import utils from nova.api.ec2 import cloud from nova.auth import manager @@ -76,12 +76,12 @@ class LibvirtConnTestCase(test.TestCase): 'bridge': 'br101', 'instance_type': 'm1.small'} - def _driver_dependent_test_setup(self): - """ - Setup method. - Call this method at the top of each testcase method, - if the testcase is necessary libvirt and cheetah. - """ + def _driver_dependant_test_setup(self): + """Call this method at the top of each testcase method. + + Checks if libvirt and cheetah, etc is installed. + Otherwise, skip testing.""" + try: global libvirt global libxml2 @@ -92,10 +92,9 @@ class LibvirtConnTestCase(test.TestCase): except ImportError, e: logging.warn("""This test has not been done since """ """using driver-dependent library Cheetah/libvirt/libxml2.""") - raise e + raise # inebitable mocks for calling - #nova.virt.libvirt_conn.LibvirtConnection.__init__ obj = utils.import_object(FLAGS.firewall_driver) fwmock = self.mox.CreateMock(obj) self.mox.StubOutWithMock(libvirt_conn, 'utils', @@ -258,51 +257,31 @@ class LibvirtConnTestCase(test.TestCase): self.assertEquals(uri, testuri) def test_get_vcpu_total(self): - """ - Check if get_vcpu_total returns appropriate cpu value - Connection/OS/driver differenct does not matter for this method, - everyone can execute for checking. - """ + """Check if get_vcpu_total returns appropriate cpu value.""" try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertTrue(0 < conn.get_vcpu_total()) - self.mox.UnsetStubs() def test_get_memory_mb_total(self): - """Check if get_memory_mb returns appropriate memory value""" + """Check if get_memory_mb returns appropriate memory value.""" try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertTrue(0 < conn.get_memory_mb_total()) - self.mox.UnsetStubs() - - def test_get_local_gb_total(self): - """Check if get_local_gb_total returns appropriate disk value""" - # Note(masumotok): leave this b/c FLAGS.instances_path is inevitable.. - #try: - # self._driver_dependent_test_setup() - #except: - # return - # - #self.mox.ReplayAll() - #conn = libvirt_conn.LibvirtConnection(False) - #self.assertTrue(0 < conn.get_local_gb_total()) - #self.mox.UnsetStubs() - pass def test_get_vcpu_used(self): - """Check if get_local_gb_total returns appropriate disk value""" + """Check if get_local_gb_total returns appropriate disk value.""" try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return @@ -321,52 +300,45 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertTrue(conn.get_vcpu_used() == 4) - self.mox.UnsetStubs() def test_get_memory_mb_used(self): - """Check if get_memory_mb returns appropriate memory value""" + """Check if get_memory_mb returns appropriate memory value.""" try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertTrue(0 < conn.get_memory_mb_used()) - self.mox.UnsetStubs() - - def test_get_local_gb_used(self): - """Check if get_local_gb_total returns appropriate disk value""" - # Note(masumotok): leave this b/c FLAGS.instances_path is inevitable - #try: - # self._driver_dependent_test_setup() - #except: - # return - - #self.mox.ReplayAll() - #conn = libvirt_conn.LibvirtConnection(False) - #self.assertTrue(0 < conn.get_local_gb_used()) - #self.mox.UnsetStubs() - pass def test_get_cpu_info_works_correctly(self): - """ - Check if get_cpu_info works correctly. - (in case libvirt.getCapabilities() works correctly) - """ - xml = ("""x86_64Nehalem""" - """Intel""" - """""" - """""" - """""" - """""" - """""" - """""" - """""") + """Check if get_cpu_info works correctly as expected.""" + xml = """ + x86_64 + Nehalem + Intel + + + + + + + + + + + + + + + + + + """ try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, @@ -376,27 +348,34 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertTrue(0 < len(conn.get_cpu_info())) - self.mox.UnsetStubs() def test_get_cpu_info_inappropreate_xml(self): - """ - Check if get_cpu_info raises exception - in case libvirt.getCapabilities() returns wrong xml - (in case of xml doesnt have tag) - """ - xml = ("""x86_64Nehalem""" - """Intel""" - """""" - """""" - """""" - """""" - """""" - """""" - """""") + """Raise exception if given xml is inappropriate.""" + xml = """ + x86_64 + Nehalem + Intel + + + + + + + + + + + + + + + + + + """ try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, @@ -409,29 +388,34 @@ class LibvirtConnTestCase(test.TestCase): conn.get_cpu_info() except exception.Invalid, e: c1 = (0 <= e.message.find('Invalid xml')) - self.assertTrue(c1) - self.mox.UnsetStubs() + self.assertTrue(c1) def test_get_cpu_info_inappropreate_xml2(self): - """ - Check if get_cpu_info raises exception - in case libvirt.getCapabilities() returns wrong xml - (in case of xml doesnt have inproper tag - meaning missing "socket" attribute) - """ - xml = ("""x86_64Nehalem""" - """Intel""" - """""" - """""" - """""" - """""" - """""" - """""" - """""") - + """Raise exception if given xml is inappropriate(topology tag).""" + + xml = """ + x86_64 + Nehalem + Intel + + + + + + + + + + + + + + + + + """ try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, @@ -444,29 +428,12 @@ class LibvirtConnTestCase(test.TestCase): conn.get_cpu_info() except exception.Invalid, e: c1 = (0 <= e.message.find('Invalid xml: topology')) - self.assertTrue(c1) - self.mox.UnsetStubs() + self.assertTrue(c1) def test_update_available_resource_works_correctly(self): - """ - In this method, vcpus/memory_mb/local_gb/vcpu_used/ - memory_mb_used/local_gb_used/hypervisor_type/ - hypervisor_version/cpu_info should be changed. - Based on this specification, this testcase confirms - if this method finishes successfully, - meaning self.db.service_update must be called with dictinary - - {'vcpu':aaa, 'memory_mb':bbb, 'local_gb':ccc, - 'vcpu_used':aaa, 'memory_mb_used':bbb, 'local_gb_sed':ccc, - 'hypervisor_type':ddd, 'hypervisor_version':eee, - 'cpu_info':fff} - - Since each value of above dict can be obtained through - driver(different depends on environment), - only dictionary keys are checked. - """ + """Confirm compute_service table is updated successfully.""" try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return @@ -478,7 +445,9 @@ class LibvirtConnTestCase(test.TestCase): host = 'foo' binary = 'nova-compute' - service_ref = {'id': 1, 'host': host, 'binary': binary, + service_ref = {'id': 1, + 'host': host, + 'binary': binary, 'topic': 'compute'} self.mox.StubOutWithMock(db, 'service_get_all_by_topic') @@ -491,15 +460,11 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) conn.update_available_resource(host) - self.mox.UnsetStubs() def test_update_resource_info_raise_exception(self): - """ - This testcase confirms if no record found on Service - table, exception can be raised. - """ + """Raise exception if no recorde found on services table.""" try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return @@ -518,18 +483,19 @@ class LibvirtConnTestCase(test.TestCase): msg = 'Cannot insert compute manager specific info' c1 = (0 <= e.message.find(msg)) self.assertTrue(c1) - self.mox.ResetAll() def test_compare_cpu_works_correctly(self): - """Calling libvirt.compute_cpu() and works correctly """ - - t = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ - """"topology":{"cores":"%s", "threads":"%s", """ - """"sockets":"%s"}, "features":[%s]}""") - cpu_info = t % ('x86', 'model', 'vendor', '2', '1', '4', '"tm"') + """Calling libvirt.compute_cpu() and works correctly.""" + t = {} + t['arch'] = 'x86' + t['model'] = 'model' + t['vendor'] = 'Intel' + t['topology'] = {'cores': "2", "threads": "1", "sockets": "4"} + t['features'] = ["tm"] + cpu_info = utils.dumps(t) try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return @@ -542,20 +508,19 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertTrue(None == conn.compare_cpu(cpu_info)) - self.mox.UnsetStubs() def test_compare_cpu_raises_exception(self): - """ - Libvirt-related exception occurs when calling - libvirt.compare_cpu(). - """ - t = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ - """"topology":{"cores":"%s", "threads":"%s", """ - """"sockets":"%s"}, "features":[%s]}""") - cpu_info = t % ('x86', 'model', 'vendor', '2', '1', '4', '"tm"') + """Libvirt-related exception occurs when calling compare_cpu().""" + t = {} + t['arch'] = 'x86' + t['model'] = 'model' + t['vendor'] = 'Intel' + t['topology'] = {'cores': "2", "threads": "1", "sockets": "4"} + t['features'] = ["tm"] + cpu_info = utils.dumps(t) try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return @@ -567,18 +532,19 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertRaises(libvirt.libvirtError, conn.compare_cpu, cpu_info) - self.mox.UnsetStubs() def test_compare_cpu_no_compatibility(self): - """libvirt.compare_cpu() return less than 0.(no compatibility)""" - - t = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ - """"topology":{"cores":"%s", "threads":"%s", """ - """"sockets":"%s"}, "features":[%s]}""") - cpu_info = t % ('x86', 'model', 'vendor', '2', '1', '4', '"tm"') + """Libvirt.compare_cpu() return less than 0.(no compatibility).""" + t = {} + t['arch'] = 'x86' + t['model'] = 'model' + t['vendor'] = 'Intel' + t['topology'] = {'cores': "2", "threads": "1", "sockets": "4"} + t['features'] = ["tm"] + cpu_info = utils.dumps(t) try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return @@ -590,16 +556,14 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) self.assertRaises(exception.Invalid, conn.compare_cpu, cpu_info) - self.mox.UnsetStubs() def test_ensure_filtering_rules_for_instance_works_correctly(self): - """ensure_filtering_rules_for_instance works as expected correctly""" - + """ensure_filtering_rules_for_instance() works successfully.""" instance_ref = models.Instance() instance_ref.__setitem__('id', 1) try: - nwmock, fwmock = self._driver_dependent_test_setup() + nwmock, fwmock = self._driver_dependant_test_setup() except: return @@ -613,16 +577,14 @@ class LibvirtConnTestCase(test.TestCase): self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) conn.ensure_filtering_rules_for_instance(instance_ref) - self.mox.UnsetStubs() def test_ensure_filtering_rules_for_instance_timeout(self): - """ensure_filtering_fules_for_instance finishes with timeout""" - + """ensure_filtering_fules_for_instance() finishes with timeout.""" instance_ref = models.Instance() instance_ref.__setitem__('id', 1) try: - nwmock, fwmock = self._driver_dependent_test_setup() + nwmock, fwmock = self._driver_dependant_test_setup() except: return @@ -642,11 +604,9 @@ class LibvirtConnTestCase(test.TestCase): except exception.Error, e: c1 = (0 <= e.message.find('Timeout migrating for')) self.assertTrue(c1) - self.mox.UnsetStubs() def test_live_migration_works_correctly(self): - """_live_migration works as expected correctly """ - + """_live_migration() works as expected correctly.""" class dummyCall(object): f = None @@ -659,7 +619,7 @@ class LibvirtConnTestCase(test.TestCase): ctxt = context.get_admin_context() try: - self._driver_dependent_test_setup() + self._driver_dependant_test_setup() except: return @@ -681,13 +641,9 @@ class LibvirtConnTestCase(test.TestCase): # Not setting post_method/recover_method in this testcase. ret = conn._live_migration(ctxt, i_ref, i_ref['host'], '', '') self.assertTrue(ret == None) - self.mox.UnsetStubs() def test_live_migration_raises_exception(self): - """ - _live_migration raises exception, then this testcase confirms - recovered method is called. - """ + """Confirms recover method is called when exceptions are raised.""" i_ref = models.Instance() i_ref.__setitem__('id', 1) i_ref.__setitem__('host', 'dummy') @@ -697,7 +653,7 @@ class LibvirtConnTestCase(test.TestCase): pass try: - nwmock, fwmock = self._driver_dependent_test_setup() + nwmock, fwmock = self._driver_dependant_test_setup() except: return @@ -724,7 +680,6 @@ class LibvirtConnTestCase(test.TestCase): conn._mlive_migration, ctxt, instance_ref, dest, '', dummy_recover_method) - self.mox.UnsetStubs() def tearDown(self): super(LibvirtConnTestCase, self).tearDown() diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 70ddd3aaf..069a424d1 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -326,59 +326,6 @@ class FakeConnection(object): 'username': 'fakeuser', 'password': 'fakepassword'} - def get_cpu_info(self): - """This method is supported only libvirt. """ - return - - def get_vcpu_total(self): - """This method is supported only libvirt. """ - return - - def get_memory_mb_total(self): - """This method is supported only libvirt. """ - return - - def get_local_gb_total(self): - """This method is supported only libvirt. """ - return - - def get_vcpu_used(self): - """This method is supported only libvirt. """ - return - - def get_memory_mb_used(self): - """This method is supported only libvirt. """ - return - - def get_local_gb_used(self): - """This method is supported only libvirt. """ - return - - def get_hypervisor_type(self): - """This method is supported only libvirt..""" - return - - def get_hypervisor_version(self): - """This method is supported only libvirt..""" - return - - def compare_cpu(self, xml): - """This method is supported only libvirt..""" - raise NotImplementedError('This method is supported only libvirt.') - - def ensure_filtering_rules_for_instance(self, instance_ref): - """This method is supported only libvirt..""" - return - - def live_migration(self, context, instance_ref, dest, - post_method, recover_method): - """This method is supported only libvirt..""" - return - - def unfilter_instance(self, instance_ref): - """This method is supported only libvirt..""" - raise NotImplementedError('This method is supported only libvirt.') - def refresh_security_group_rules(self, security_group_id): """This method is called after a change to security groups. @@ -428,20 +375,25 @@ class FakeConnection(object): return True def update_available_resource(self, ctxt, host): - """This method is supported only libvirt. """ + """This method is supported only by libvirt.""" return def compare_cpu(self, xml): - """This method is supported only libvirt..""" - raise NotImplementedError('This method is supported only libvirt.') + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') def ensure_filtering_rules_for_instance(self, instance_ref): - """This method is supported only libvirt..""" - raise NotImplementedError('This method is supported only libvirt.') + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') + + def live_migration(self, context, instance_ref, dest, + post_method, recover_method): + """This method is supported only by libvirt.""" + return - def live_migration(self, context, instance_ref, dest): - """This method is supported only libvirt..""" - raise NotImplementedError('This method is supported only libvirt.') + def unfilter_instance(self, instance_ref): + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') class FakeInstance(object): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index d39836e72..934aed960 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -36,7 +36,6 @@ Supports KVM, QEMU, UML, and XEN. """ -import json import os import shutil import random @@ -105,7 +104,7 @@ flags.DEFINE_string('firewall_driver', 'Firewall driver (defaults to iptables)') flags.DEFINE_string('cpuinfo_xml_template', utils.abspath('virt/cpuinfo.xml.template'), - 'CpuInfo XML Template (used only live migration now)') + 'CpuInfo XML Template (Used only live migration now)') flags.DEFINE_string('live_migration_uri', "qemu+tcp://%s/system", 'Define protocol used by live_migration feature') @@ -851,23 +850,46 @@ class LibvirtConnection(object): return interfaces def get_vcpu_total(self): - """ Get vcpu number of physical computer. """ + """Get vcpu number of physical computer. + + :returns: the number of cpu core. + + """ + return open('/proc/cpuinfo').read().count('processor') def get_memory_mb_total(self): - """Get the total memory size(MB) of physical computer .""" + """Get the total memory size(MB) of physical computer. + + :returns: the total amount of memory(MB). + + """ + meminfo = open('/proc/meminfo').read().split() idx = meminfo.index('MemTotal:') # transforming kb to mb. return int(meminfo[idx + 1]) / 1024 def get_local_gb_total(self): - """Get the total hdd size(GB) of physical computer .""" + """Get the total hdd size(GB) of physical computer. + + :returns: + The total amount of HDD(GB). + Note that this value shows a partition where + NOVA-INST-DIR/instances mounts. + + """ + hddinfo = os.statvfs(FLAGS.instances_path) return hddinfo.f_frsize * hddinfo.f_blocks / 1024 / 1024 / 1024 def get_vcpu_used(self): - """ Get vcpu available number of physical computer. """ + """ Get vcpu usage number of physical computer. + + :returns: The total number of vcpu that currently used. + + """ + total = 0 for i in self._conn.listDomainsID(): dom = self._conn.lookupByID(i) @@ -875,7 +897,12 @@ class LibvirtConnection(object): return total def get_memory_mb_used(self): - """Get the free memory size(MB) of physical computer.""" + """Get the free memory size(MB) of physical computer. + + :returns: the total usage of memory(MB). + + """ + m = open('/proc/meminfo').read().split() idx1 = m.index('MemFree:') idx2 = m.index('Buffers:') @@ -884,21 +911,47 @@ class LibvirtConnection(object): return self.get_memory_mb_total() - avail def get_local_gb_used(self): - """Get the free hdd size(GB) of physical computer .""" + """Get the free hdd size(GB) of physical computer. + + :returns: + The total usage of HDD(GB). + Note that this value shows a partition where + NOVA-INST-DIR/instances mounts. + + """ + hddinfo = os.statvfs(FLAGS.instances_path) avail = hddinfo.f_frsize * hddinfo.f_bavail / 1024 / 1024 / 1024 return self.get_local_gb_total() - avail def get_hypervisor_type(self): - """ Get hypervisor type """ + """Get hypervisor type. + + :returns: hypervisor type (ex. qemu) + + """ + return self._conn.getType() def get_hypervisor_version(self): - """ Get hypervisor version """ + """Get hypervisor version. + + :returns: hypervisor version (ex. 12003) + + """ + return self._conn.getVersion() def get_cpu_info(self): - """ Get cpuinfo information """ + """Get cpuinfo information. + + Obtains cpu feature from virConnect.getCapabilities, + and returns as a json string. + + :return: see above description + + """ + xml = self._conn.getCapabilities() xml = libxml2.parseDoc(xml) nodes = xml.xpathEval('//cpu') @@ -931,17 +984,9 @@ class LibvirtConnection(object): for nodes in feature_nodes: features.append(nodes.get_properties().getContent()) - template = ("""{"arch":"%s", "model":"%s", "vendor":"%s", """ - """"topology":{"cores":"%s", "threads":"%s", """ - """"sockets":"%s"}, "features":[%s]}""") - f = ['"%s"' % x for x in features] - return template % (cpu_info['arch'], - cpu_info['model'], - cpu_info['vendor'], - topology['cores'], - topology['sockets'], - topology['threads'], - ', '.join(f)) + cpu_info['topology'] = topology + cpu_info['features'] = features + return utils.dumps(cpu_info) def block_stats(self, instance_name, disk): """ @@ -974,12 +1019,16 @@ class LibvirtConnection(object): self.firewall_driver.refresh_security_group_members(security_group_id) def update_available_resource(self, ctxt, host): - """ - Update compute manager resource info on Service table. + """Updates compute manager resource info on ComputeService table. + This method is called when nova-coompute launches, and whenever admin executes "nova-manage service update_resource". + :param ctxt: security context + :param host: hostname that compute manager is currently running + """ + try: service_ref = db.service_get_all_compute_by_host(ctxt, host)[0] except exception.NotFound: @@ -1008,44 +1057,44 @@ class LibvirtConnection(object): db.compute_service_update(ctxt, compute_service_ref[0]['id'], dic) def compare_cpu(self, cpu_info): - """ - Check the host cpu is compatible to a cpu given by xml. + """Checks the host cpu is compatible to a cpu given by xml. + "xml" must be a part of libvirt.openReadonly().getCapabilities(). return values follows by virCPUCompareResult. if 0 > return value, do live migration. - 'http://libvirt.org/html/libvirt-libvirt.html#virCPUCompareResult' + + :param cpu_info: json string that shows cpu feature(see get_cpu_info()) + :returns: + None. if given cpu info is not compatible to this server, + raise exception. + """ - msg = _('Checking cpu_info: instance was launched this cpu.\n: %s ') - LOG.info(msg % cpu_info) - dic = json.loads(cpu_info) - xml = str(Template(self.cpuinfo_xml, searchList=dic)) - msg = _('to xml...\n: %s ') - LOG.info(msg % xml) - url = 'http://libvirt.org/html/libvirt-libvirt.html' - url += '#virCPUCompareResult\n' - msg = 'CPU does not have compativility.\n' - msg += 'result:%s \n' - msg += 'Refer to %s' - msg = _(msg) + LOG.info(_('Checking cpu_info: instance was launched this cpu.\n%s') + % cpu_info) + dic = utils.loads(cpu_info) + xml = str(Template(self.cpuinfo_xml, searchList=dic)) + LOG.info(_('to xml...\n:%s ' % xml)) + u = "http://libvirt.org/html/libvirt-libvirt.html#virCPUCompareResult" + m = _("CPU doesn't have compatibility.\n\n%(ret)s\n\nRefer to %(u)s") # unknown character exists in xml, then libvirt complains try: ret = self._conn.compareCPU(xml, 0) except libvirt.libvirtError, e: - LOG.error(msg % (e.message, url)) - raise e + ret = e.message + LOG.error(m % locals()) + raise if ret <= 0: - raise exception.Invalid(msg % (ret, url)) + raise exception.Invalid(m % locals()) return def ensure_filtering_rules_for_instance(self, instance_ref): - """ - Setting up inevitable filtering rules on compute node, - and waiting for its completion. + """Setting up filtering rules and waiting for its completion. + To migrate an instance, filtering rules to hypervisors and firewalls are inevitable on destination host. ( Waiting only for filterling rules to hypervisor, @@ -1062,9 +1111,12 @@ class LibvirtConnection(object): Don't use thread for this method since migration should not be started when setting-up filtering rules operations are not completed. + + :params instance_ref: nova.db.sqlalchemy.models.Instance object + """ - # Tf any instances never launch at destination host, + # If any instances never launch at destination host, # basic-filtering must be set here. self.firewall_driver.setup_basic_filtering(instance_ref) # setting up n)ova-instance-instance-xx mainly. @@ -1088,16 +1140,42 @@ class LibvirtConnection(object): def live_migration(self, ctxt, instance_ref, dest, post_method, recover_method): + """Spawning live_migration operation for distributing high-load. + + :params ctxt: security context + :params instance_ref: + nova.db.sqlalchemy.models.Instance object + instance object that is migrated. + :params dest: destination host + :params post_method: + post operation method. + expected nova.compute.manager.post_live_migration. + :params recover_method: + recovery method when any exception occurs. + expected nova.compute.manager.recover_live_migration. + """ - Just spawning live_migration operation for - distributing high-load. - """ + greenthread.spawn(self._live_migration, ctxt, instance_ref, dest, post_method, recover_method) def _live_migration(self, ctxt, instance_ref, dest, post_method, recover_method): - """ Do live migration.""" + """Do live migration. + + :params ctxt: security context + :params instance_ref: + nova.db.sqlalchemy.models.Instance object + instance object that is migrated. + :params dest: destination host + :params post_method: + post operation method. + expected nova.compute.manager.post_live_migration. + :params recover_method: + recovery method when any exception occurs. + expected nova.compute.manager.recover_live_migration. + + """ # Do live migration. try: @@ -1122,7 +1200,7 @@ class LibvirtConnection(object): except Exception, e: recover_method(ctxt, instance_ref) - raise e + raise # Waiting for completion of live_migration. timer = utils.LoopingCall(f=None) @@ -1139,7 +1217,7 @@ class LibvirtConnection(object): timer.start(interval=0.5, now=True) def unfilter_instance(self, instance_ref): - """See comments of same method in firewall_driver""" + """See comments of same method in firewall_driver.""" self.firewall_driver.unfilter_instance(instance_ref) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 3dd7d6e94..0e12a4587 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -231,25 +231,25 @@ class XenAPIConnection(object): 'password': FLAGS.xenapi_connection_password} def update_available_resource(self, ctxt, host): - """This method is supported only libvirt. """ + """This method is supported only by libvirt.""" return def compare_cpu(self, xml): - """This method is supported only libvirt..""" - raise NotImplementedError('This method is supported only libvirt.') + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') def ensure_filtering_rules_for_instance(self, instance_ref): - """This method is supported only libvirt..""" + """This method is supported only libvirt.""" return def live_migration(self, context, instance_ref, dest, post_method, recover_method): - """This method is supported only libvirt..""" + """This method is supported only by libvirt.""" return def unfilter_instance(self, instance_ref): - """This method is supported only libvirt..""" - raise NotImplementedError('This method is supported only libvirt.') + """This method is supported only by libvirt.""" + raise NotImplementedError('This method is supported only by libvirt.') class XenAPISession(object): diff --git a/nova/volume/driver.py b/nova/volume/driver.py index b74c74197..fbc52a598 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -140,7 +140,7 @@ class VolumeDriver(object): def check_for_export(self, context, volume_id): """Make sure volume is exported.""" - return True + raise NotImplementedError() class AOEDriver(VolumeDriver): @@ -229,9 +229,9 @@ class AOEDriver(VolumeDriver): break if not exported: # Instance will be terminated in this case. - desc = _("""Cannot confirm exported volume id:%(volume_id)s.""" - """vblade process for e%(shelf_id)s.%(blade_id)s """ - """isn't running.""") % locals() + desc = _("Cannot confirm exported volume id:%(volume_id)s." + "vblade process for e%(shelf_id)s.%(blade_id)s " + "isn't running.") % locals() raise exception.ProcessExecutionError(out, _err, cmd=cmd, description=desc) @@ -373,9 +373,9 @@ class ISCSIDriver(VolumeDriver): # Instances remount read-only in this case. # /etc/init.d/iscsitarget restart and rebooting nova-volume # is better since ensure_export() works at boot time. - logging.error(_("""Cannot confirm exported volume """ - """id:%(volume_id)s.""") % locals()) - raise e + logging.error(_("Cannot confirm exported volume " + "id:%(volume_id)s.") % locals()) + raise class FakeISCSIDriver(ISCSIDriver): -- cgit From c6b2d07f47004576fa386a6d270203b1d7937664 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Wed, 23 Feb 2011 00:15:39 +0900 Subject: Fix tiny mitakes! (remove unnecessary comment, etc) --- nova/tests/test_scheduler.py | 2 +- nova/tests/test_volume.py | 3 --- nova/virt/libvirt_conn.py | 2 +- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 301106848..47a6d0e82 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -258,10 +258,10 @@ class SimpleDriverTestCase(test.TestCase): inst['project_id'] = self.project.id inst['instance_type'] = 'm1.tiny' inst['mac_address'] = utils.generate_mac() + inst['vcpus'] = kwargs.get('vcpus', 1) inst['ami_launch_index'] = 0 inst['availability_zone'] = kwargs.get('availability_zone', None) inst['host'] = kwargs.get('host', 'dummy') - inst['vcpus'] = kwargs.get('vcpus', 4) inst['memory_mb'] = kwargs.get('memory_mb', 20) inst['local_gb'] = kwargs.get('local_gb', 30) inst['launched_on'] = kwargs.get('launghed_on', 'dummy') diff --git a/nova/tests/test_volume.py b/nova/tests/test_volume.py index 6ae075caa..e8b4ceee8 100644 --- a/nova/tests/test_volume.py +++ b/nova/tests/test_volume.py @@ -318,9 +318,6 @@ class ISCSITestCase(DriverTestCase): mountpoint = "/dev/sd" + chr((ord('b') + index)) db.volume_attached(self.context, vol_ref['id'], self.instance_id, mountpoint) - #iscsi_target = db.volume_allocate_iscsi_target(self.context, - # vol_ref['id'], - # vol_ref['host']) volume_id_list.append(vol_ref['id']) return volume_id_list diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 934aed960..118ea13e5 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -888,7 +888,7 @@ class LibvirtConnection(object): :returns: The total number of vcpu that currently used. - """ + """ total = 0 for i in self._conn.listDomainsID(): -- cgit From 485a6c5a9502679bc5ecf02f8e758170ac0335dc Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Wed, 23 Feb 2011 01:20:39 +0900 Subject: Fixed some docstring --- nova/scheduler/manager.py | 2 +- nova/tests/test_scheduler.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 783594c6f..cd5012fd5 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -59,7 +59,7 @@ class SchedulerManager(manager.Manager): try: host = getattr(self.driver, driver_method)(elevated, *args, **kwargs) - except AttributeError, e: + except AttributeError: host = self.driver.schedule(elevated, topic, *args, **kwargs) rpc.cast(context, diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 47a6d0e82..71e524bca 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -60,6 +60,7 @@ class SchedulerTestCase(test.TestCase): self.flags(scheduler_driver='nova.tests.test_scheduler.TestDriver') def _create_compute_service(self): + """Create compute-manager(ComputeService and Service record).""" ctxt = context.get_admin_context() dic = {'host': 'dummy', 'binary': 'nova-compute', 'topic': 'compute', 'report_count': 0, 'availability_zone': 'dummyzone'} @@ -150,7 +151,7 @@ class SchedulerTestCase(test.TestCase): db.service_destroy(ctxt, s_ref['id']) def test_show_host_resources_works_correctly(self): - """show_host_resources() works correctly as expected.""" + """Show_host_resources() works correctly as expected.""" scheduler = manager.SchedulerManager() ctxt = context.get_admin_context() -- cgit From 493aa34da71e7dc3c28c6a55254b6d7ed4d81b72 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Sat, 26 Feb 2011 22:13:28 +0100 Subject: beautification... --- bin/nova-manage | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 68847bdde..2e4e091cf 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -623,8 +623,8 @@ class InstanceCommands(object): def list(self, host=None, instance=None): """Show a list of all instances""" - print "%-10s %-15s %-10s %-10s %-19s %-12s %-12s" \ - " %-12s %-10s %-10s %-10s %-5s" % ( + print "%-10s %-15s %-10s %-10s %-19s %-12s %-12s %-12s" \ + " %-10s %-10s %-10s %-5s" % ( _('instance'), _('node'), _('type'), @@ -636,7 +636,7 @@ class InstanceCommands(object): _('project'), _('user'), _('zone'), - _('index'), + _('index') ) for instance in db.instance_get_all(context.get_admin_context()): @@ -716,8 +716,8 @@ CATEGORIES = [ ('service', ServiceCommands), ('log', LogCommands), ('db', DbCommands), - ('volume', VolumeCommands) - ('instance', InstanceCommands), + ('volume', VolumeCommands), + ('instance', InstanceCommands) ] -- cgit From 2714b2df0d21ecb08966c4d145d2d75fa1bb201d Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Sun, 27 Feb 2011 00:07:03 +0100 Subject: fixed FIXME --- nova/db/sqlalchemy/api.py | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index d07c53759..828d24c78 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -598,17 +598,11 @@ def fixed_ip_get_all(context, session=None): def fixed_ip_get_all_by_host(context, host=None): session = get_session() - # FIXME: I'm sure that SQLAlchemy can handle this in a nicer way - instances = session.query(models.Instance).\ - filter_by(state=1).\ - filter_by(host=host).\ - all() - - result = [] - for instance in instances: - result.append(session.query(models.FixedIp).\ - filter_by(instance_id=instance['id']).\ - first()) + result = session.query(models.FixedIp).\ + join(models.FixedIp.instance).\ + filter_by(state=1).\ + filter_by(host=host).\ + all() if not result: raise exception.NotFound(_('No fixed ips for this host defined')) -- cgit From f72e5b618387a7b5a06f0e5b7e68af51c6667327 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Sun, 27 Feb 2011 00:13:05 +0100 Subject: added listing of instances running on a specific host chronos:~ # nova-manage fixed list ares network IP address MAC address hostname host 192.168.3.0/24 192.168.3.6 02:16:3e:75:d7:9a i-00000c1c ares --- bin/nova-manage | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index 2e4e091cf..12ccfa0e9 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -639,7 +639,13 @@ class InstanceCommands(object): _('index') ) - for instance in db.instance_get_all(context.get_admin_context()): + if host == None: + instances = db.instance_get_all(context.get_admin_context()) + else: + instances = db.instance_get_all_by_host( + context.get_admin_context(), host) + + for instance in instances: print "%-10s %-15s %-10s %-10s %-19s %-12s %-12s %-12s" \ " %-10s %-10s %-10s %-5d" % ( instance['hostname'], -- cgit From 0550124fcd863be60dd0e6fefb5f30641331b198 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Mon, 28 Feb 2011 18:06:11 -0500 Subject: add test for instance creation without personalities --- nova/tests/api/openstack/test_servers.py | 40 ++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 7a25abe9d..7e5bc0080 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -232,6 +232,46 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) + def _create_instance_with_personality(self, personality): + + class FakeComputeAPI(object): + + def __init__(self): + self.onset_files = None + + def create(*args, **kwargs): + if 'onset_files' in kwargs: + self.onset_files = kwargs['onset_files'] + else: + self.onset_files = None + return [{'id': '1234', 'display_name': 'fakeinstance'}] + + def make_stub_method(canned_return): + def stub_method(*args, **kwargs): + return canned_return + return stub_method + + compute_api = FakeComputeAPI() + self.stubs.Set(nova.compute, 'API', make_stub_method(compute_api)) + self.stubs.Set(nova.api.openstack.servers.Controller, + '_get_kernel_ramdisk_from_image', make_stub_method((1, 1))) + self.stubs.Set(nova.api.openstack.common, + 'get_image_id_from_image_hash', make_stub_method(2)) + body = dict(server=dict( + name='server_test', imageId=2, flavorId=2, + metadata={}, + personality=personality)) + + req = webob.Request.blank('/v1.0/servers') + req.method = 'POST' + req.body = json.dumps(body) + return req.get_response(fakes.wsgi_app()), compute_api.onset_files + + def test_create_instance_with_no_personality(self): + res, onset_files = self._create_instance_with_personality(personality={}) + self.assertEquals(res.status_int, 200) + self.assertEquals(onset_files, None) + def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' -- cgit From 1caa7f189827b4721c2e9d3ddf753acd749d7916 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Tue, 1 Mar 2011 17:52:46 +0900 Subject: rename db migration script --- nova/compute/manager.py | 2 +- .../versions/007_add_live_migration.py | 83 ++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 nova/db/sqlalchemy/migrate_repo/versions/007_add_live_migration.py diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 8b90ffbca..d085a0b6a 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -818,7 +818,7 @@ class ComputeManager(manager.Manager): LOG.info(_('Migrating %(i_name)s to %(dest)s finishes successfully.') % locals()) - LOG.info(_("The below error is normally occurs." + LOG.info(_("The below error is normally occurs. " "Just check if instance is successfully migrated.\n" "libvir: QEMU error : Domain not found: no domain " "with matching name..")) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/007_add_live_migration.py b/nova/db/sqlalchemy/migrate_repo/versions/007_add_live_migration.py new file mode 100644 index 000000000..2689b5b74 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/007_add_live_migration.py @@ -0,0 +1,83 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from migrate import * +from nova import log as logging +from sqlalchemy import * + + +meta = MetaData() + +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +# +# New Tables +# + +compute_services = Table('compute_services', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('service_id', Integer(), nullable=False), + + Column('vcpus', Integer(), nullable=False), + Column('memory_mb', Integer(), nullable=False), + Column('local_gb', Integer(), nullable=False), + Column('vcpus_used', Integer(), nullable=False), + Column('memory_mb_used', Integer(), nullable=False), + Column('local_gb_used', Integer(), nullable=False), + Column('hypervisor_type', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=False), + Column('hypervisor_version', Integer(), nullable=False), + Column('cpu_info', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=False), + ) + + +# +# Tables to alter +# +instances_launched_on = Column( + 'launched_on', + Text(convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=True) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + try: + compute_services.create() + except Exception: + logging.info(repr(compute_services)) + logging.exception('Exception while creating table') + meta.drop_all(tables=[compute_services]) + raise + + instances.create_column(instances_launched_on) -- cgit From d13a623625a56a029f9dd5ccba3e70f492efdb2c Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Tue, 1 Mar 2011 18:32:57 +0900 Subject: test_compute is changed b/c lack of import instance_types --- nova/tests/test_compute.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 3c88d186d..2a18dd47b 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -32,6 +32,7 @@ from nova import rpc from nova import test from nova import utils from nova.auth import manager +from nova.compute import instance_types from nova.compute import manager as compute_manager from nova.compute import power_state from nova.db.sqlalchemy import models -- cgit From 688acacd85e07fc578c8731df6a4421e64499c8b Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Tue, 1 Mar 2011 18:53:02 +0900 Subject: At previous commit, I forget to erase conflict - fixed it. --- nova/tests/test_compute.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 75fbc9324..3438719f4 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -309,7 +309,13 @@ class ComputeTestCase(test.TestCase): self.compute.terminate_instance(self.context, instance_id) -<<<<<<< TREE + def test_get_by_flavor_id(self): + type = instance_types.get_by_flavor_id(1) + self.assertEqual(type, 'm1.tiny') + + type = instance_types.get_by_flavor_id("1") + self.assertEqual(type, 'm1.tiny') + def _setup_other_managers(self): self.volume_manager = utils.import_object(FLAGS.volume_manager) self.network_manager = utils.import_object(FLAGS.network_manager) @@ -562,11 +568,3 @@ class ComputeTestCase(test.TestCase): db.instance_destroy(c, instance_id) db.volume_destroy(c, v_ref['id']) db.floating_ip_destroy(c, flo_addr) -======= - def test_get_by_flavor_id(self): - type = instance_types.get_by_flavor_id(1) - self.assertEqual(type, 'm1.tiny') - - type = instance_types.get_by_flavor_id("1") - self.assertEqual(type, 'm1.tiny') ->>>>>>> MERGE-SOURCE -- cgit From f9d08c16d5c620c711d962a78be3a94b99364f14 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 1 Mar 2011 13:56:33 -0500 Subject: support adding a single personality in the osapi --- nova/api/openstack/servers.py | 22 ++++++++++++++++++++-- nova/tests/api/openstack/test_servers.py | 21 ++++++++++++++++----- 2 files changed, 36 insertions(+), 7 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 73c7bfe17..92e5c9024 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -80,7 +80,6 @@ def _translate_detail_keys(inst): return dict(server=inst_dict) - def _translate_keys(inst): """ Coerces into dictionary format, excluding all model attributes save for id and name """ @@ -154,6 +153,22 @@ class Controller(wsgi.Controller): image = self._image_service.show(req.environ['nova.context'], image_id) return lookup('kernel_id'), lookup('ramdisk_id') + + def _get_onset_files_from_personality_attr(self, personality_attr): + """ + Create a list of onset files from the personality request attribute + + At this time, onset_files must be formatted as a list of + (file_path, file_content) pairs for compatibility with the + underlying compute service. + """ + onset_files = [] + for personality in personality_attr: + path = personality['path'] + contents = personality['contents'] + onset_files.append((path, contents)) + return onset_files + def create(self, req): """ Creates a new server for a given user """ env = self._deserialize(req.body, req) @@ -181,6 +196,9 @@ class Controller(wsgi.Controller): for k, v in env['server']['metadata'].items(): metadata.append({'key': k, 'value': v}) + personality = env['server'].get('personality', []) + onset_files = self._get_onset_files_from_personality_attr(personality) + instances = self.compute_api.create( context, instance_types.get_by_flavor_id(env['server']['flavorId']), @@ -192,7 +210,7 @@ class Controller(wsgi.Controller): key_name=key_pair['name'], key_data=key_pair['public_key'], metadata=metadata, - onset_files=env.get('onset_files', [])) + onset_files=onset_files) return _translate_keys(instances[0]) def update(self, req, id): diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 7e5bc0080..42665ba6d 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -232,6 +232,9 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) + def _personality_dict(self, path, contents): + return {'path': path, 'contents': contents} + def _create_instance_with_personality(self, personality): class FakeComputeAPI(object): @@ -239,7 +242,7 @@ class ServersTest(test.TestCase): def __init__(self): self.onset_files = None - def create(*args, **kwargs): + def create(self, *args, **kwargs): if 'onset_files' in kwargs: self.onset_files = kwargs['onset_files'] else: @@ -265,12 +268,20 @@ class ServersTest(test.TestCase): req = webob.Request.blank('/v1.0/servers') req.method = 'POST' req.body = json.dumps(body) - return req.get_response(fakes.wsgi_app()), compute_api.onset_files + return req, req.get_response(fakes.wsgi_app()), compute_api.onset_files def test_create_instance_with_no_personality(self): - res, onset_files = self._create_instance_with_personality(personality={}) - self.assertEquals(res.status_int, 200) - self.assertEquals(onset_files, None) + request, response, onset_files = \ + self._create_instance_with_personality(personality=[]) + self.assertEquals(response.status_int, 200) + self.assertEquals(onset_files, []) + + def test_create_instance_with_one_personality(self): + personality = [self._personality_dict('/my/path', 'myfilecontents')] + request, response, onset_files = \ + self._create_instance_with_personality(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(onset_files, [('/my/path', 'myfilecontents')]) def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') -- cgit From 94e42c3002f9043fc3c5b90a1cb5ad0c50ba261b Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 1 Mar 2011 16:21:37 -0500 Subject: ensure personality contents are b64 encoded --- nova/api/openstack/servers.py | 15 ++++++++++----- nova/tests/api/openstack/test_servers.py | 21 +++++++++++++++++---- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 8491fe697..8908bbdca 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +import base64 import hashlib import json import traceback @@ -138,7 +139,7 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def _get_onset_files_from_personality_attr(self, personality_attr): + def _get_onset_files_from_personality(self, personality): """ Create a list of onset files from the personality request attribute @@ -147,9 +148,13 @@ class Controller(wsgi.Controller): underlying compute service. """ onset_files = [] - for personality in personality_attr: - path = personality['path'] - contents = personality['contents'] + for item in personality: + path = item['path'] + try: + contents = base64.b64decode(item['contents']) + except TypeError: + raise exc.HTTPBadRequest(explanation= + 'Personality content for %s cannot be decoded' % path) onset_files.append((path, contents)) return onset_files @@ -181,7 +186,7 @@ class Controller(wsgi.Controller): metadata.append({'key': k, 'value': v}) personality = env['server'].get('personality', []) - onset_files = self._get_onset_files_from_personality_attr(personality) + onset_files = self._get_onset_files_from_personality(personality) instances = self.compute_api.create( context, diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 144dbd4af..4b40793a7 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -15,6 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. +import base64 import datetime import json @@ -272,16 +273,28 @@ class ServersTest(test.TestCase): def test_create_instance_with_no_personality(self): request, response, onset_files = \ - self._create_instance_with_personality(personality=[]) + self._create_instance_with_personality(personality=[]) self.assertEquals(response.status_int, 200) self.assertEquals(onset_files, []) - def test_create_instance_with_one_personality(self): - personality = [self._personality_dict('/my/path', 'myfilecontents')] + def test_create_instance_with_personality(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Hello, World!"\n' + b64contents = base64.b64encode(contents) + personality = [self._personality_dict(path, b64contents)] request, response, onset_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [('/my/path', 'myfilecontents')]) + self.assertEquals(onset_files, [(path, contents)]) + + def test_create_instance_with_personality_with_non_b64_content(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Oh no!"\n' + personality = [self._personality_dict(path, contents)] + request, response, onset_files = \ + self._create_instance_with_personality(personality) + self.assertEquals(response.status_int, 400) + self.assertEquals(onset_files, None) def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') -- cgit From 7b3ccd5fd1636ebc437a89a3667e6e712004e87f Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 1 Mar 2011 16:48:01 -0500 Subject: test osapi server create with multiple personalities --- nova/tests/api/openstack/test_servers.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 4b40793a7..dd951e90c 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -296,6 +296,20 @@ class ServersTest(test.TestCase): self.assertEquals(response.status_int, 400) self.assertEquals(onset_files, None) + def test_create_instance_with_two_personalities(self): + files = [ + ('/etc/sudoers', 'ALL ALL=NOPASSWD: ALL\n'), + ('/etc/motd', 'Enjoy your root access!\n'), + ] + personality = [] + for path, content in files: + personality.append(self._personality_dict( + path, base64.b64encode(content))) + request, response, onset_files = \ + self._create_instance_with_personality(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(onset_files, files) + def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' -- cgit From d33866923958b3529a812f4eef7dea4a6591a423 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 2 Mar 2011 17:36:21 -0500 Subject: add support for quotas on file injection --- nova/quota.py | 29 +++++++++++++++++++++++------ nova/tests/test_quota.py | 10 ++++++++++ 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/nova/quota.py b/nova/quota.py index 6b52a97fa..14b388794 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -37,15 +37,22 @@ flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') flags.DEFINE_integer('quota_metadata_items', 128, 'number of metadata items allowed per instance') +flags.DEFINE_integer('quota_file_injection_max_files', 5, + 'number of files allowed during file injection') +flags.DEFINE_integer('quota_file_injection_max_file_bytes', 10 * 1024, + 'number of bytes allowed per file during file injection') def get_quota(context, project_id): - rval = {'instances': FLAGS.quota_instances, - 'cores': FLAGS.quota_cores, - 'volumes': FLAGS.quota_volumes, - 'gigabytes': FLAGS.quota_gigabytes, - 'floating_ips': FLAGS.quota_floating_ips, - 'metadata_items': FLAGS.quota_metadata_items} + rval = { + 'instances': FLAGS.quota_instances, + 'cores': FLAGS.quota_cores, + 'volumes': FLAGS.quota_volumes, + 'gigabytes': FLAGS.quota_gigabytes, + 'floating_ips': FLAGS.quota_floating_ips, + 'metadata_items': FLAGS.quota_metadata_items, + } + try: quota = db.quota_get(context, project_id) for key in rval.keys(): @@ -106,6 +113,16 @@ def allowed_metadata_items(context, num_metadata_items): return min(num_metadata_items, num_allowed_metadata_items) +def allowed_file_injection_files(context): + """Return the number of files allowed per file injection""" + return FLAGS.quota_file_injection_max_files + + +def allowed_file_injection_file_bytes(context): + """Return the number of bytes allowed per file during injection""" + return FLAGS.quota_file_injection_max_file_bytes + + class QuotaError(exception.ApiError): """Quota Exceeeded""" pass diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 1e42fddf3..48e5a5538 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -176,3 +176,13 @@ class QuotaTestCase(test.TestCase): instance_type='m1.small', image_id='fake', metadata=metadata) + + def test_allowed_file_injection_files(self): + self.assertEqual( + quota.allowed_file_injection_files(self.context), + FLAGS.quota_file_injection_max_files) + + def test_allowed_file_injection_file_bytes(self): + self.assertEqual( + quota.allowed_file_injection_file_bytes(self.context), + FLAGS.quota_file_injection_max_file_bytes) -- cgit From 668cdc96b3f6fb412b9d1d4a3780744d6b2340b1 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 3 Mar 2011 00:59:09 -0500 Subject: more rigorous testing and error handling for os api personality --- nova/api/openstack/servers.py | 8 ++++++-- nova/tests/api/openstack/test_servers.py | 25 +++++++++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 8908bbdca..73c787828 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -149,9 +149,13 @@ class Controller(wsgi.Controller): """ onset_files = [] for item in personality: - path = item['path'] try: - contents = base64.b64decode(item['contents']) + path = item['path'] + contents = item['contents'] + except TypeError: + raise exc.HTTPBadRequest(explanation='Bad personality format') + try: + contents = base64.b64decode(contents) except TypeError: raise exc.HTTPBadRequest(explanation= 'Personality content for %s cannot be decoded' % path) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index dd951e90c..272d34e3a 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -265,6 +265,8 @@ class ServersTest(test.TestCase): name='server_test', imageId=2, flavorId=2, metadata={}, personality=personality)) + if personality is None: + del body['server']['personality'] req = webob.Request.blank('/v1.0/servers') req.method = 'POST' @@ -273,7 +275,7 @@ class ServersTest(test.TestCase): def test_create_instance_with_no_personality(self): request, response, onset_files = \ - self._create_instance_with_personality(personality=[]) + self._create_instance_with_personality(personality=None) self.assertEquals(response.status_int, 200) self.assertEquals(onset_files, []) @@ -296,10 +298,11 @@ class ServersTest(test.TestCase): self.assertEquals(response.status_int, 400) self.assertEquals(onset_files, None) - def test_create_instance_with_two_personalities(self): + def test_create_instance_with_three_personalities(self): files = [ ('/etc/sudoers', 'ALL ALL=NOPASSWD: ALL\n'), ('/etc/motd', 'Enjoy your root access!\n'), + ('/etc/dovecot.conf', 'dovecot\nconfig\nstuff\n'), ] personality = [] for path, content in files: @@ -310,6 +313,24 @@ class ServersTest(test.TestCase): self.assertEquals(response.status_int, 200) self.assertEquals(onset_files, files) + def test_create_instance_personality_empty_content(self): + path = '/my/file/path' + contents = '' + personality = [self._personality_dict(path, contents)] + request, response, onset_files = \ + self._create_instance_with_personality(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(onset_files, [(path, contents)]) + + def test_create_instance_personality_not_a_list(self): + path = '/my/file/path' + contents = 'myfilecontents' + personality = self._personality_dict(path, contents) + request, response, onset_files = \ + self._create_instance_with_personality(personality) + self.assertEquals(response.status_int, 400) + self.assertEquals(onset_files, None) + def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' -- cgit From 693e4335dbef72317147abd70bdaa10e0d174020 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Thu, 3 Mar 2011 22:54:11 +0900 Subject: Fixed based on reviewer's comments. Main changes are below. 1. Rename nova.compute.manager.ComputeManager.mktmpfile for better naming. 2. Several tests code in tests/test_virt.py are removed. Because it only works in libvirt environment. Only db-related testcode remains. --- nova/compute/manager.py | 53 ++++--- nova/scheduler/driver.py | 74 +++++---- nova/scheduler/manager.py | 18 +-- nova/tests/test_scheduler.py | 14 +- nova/tests/test_virt.py | 363 +++++++++---------------------------------- nova/virt/libvirt_conn.py | 10 +- nova/volume/manager.py | 4 +- 7 files changed, 166 insertions(+), 370 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index d085a0b6a..7104daa1e 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -624,11 +624,12 @@ class ComputeManager(manager.Manager): return self.driver.compare_cpu(cpu_info) @exception.wrap_exception - def mktmpfile(self, context): + def create_shared_storage_test_file(self, context): """Makes tmpfile under FLAGS.instance_path. This method enables compute nodes to recognize that they mounts - same shared storage. mktmpfile()/confirm_tmpfile is a pair. + same shared storage. (create|check|creanup)_shared_storage_test_file() + is a pair. :param context: security context :returns: tmpfile name(basename) @@ -636,26 +637,36 @@ class ComputeManager(manager.Manager): """ dirpath = FLAGS.instances_path - fd, name = tempfile.mkstemp(dir=dirpath) + fd, tmp_file = tempfile.mkstemp(dir=dirpath) LOG.debug(_("Creating tmpfile %s to notify to other " - "compute node that they mounts same storage.") % name) + "compute node that they mounts same storage.") % tmp_file) os.fdopen(fd, 'w+').close() - return os.path.basename(name) + return os.path.basename(tmp_file) @exception.wrap_exception - def confirm_tmpfile(self, context, filename): - """Confirms existence of the tmpfile given by path. + def check_shared_storage_test_file(self, context, filename): + """Confirms existence of the tmpfile under FLAGS.instances_path. :param context: security context :param filename: confirm existence of FLAGS.instances_path/thisfile - :returns: depends on os.remove() """ - p = os.path.join(FLAGS.instances_path, filename) - if not os.path.exists(p): - raise exception.NotFound(_('%s not found') % p) - return os.remove(p) + tmp_file = os.path.join(FLAGS.instances_path, filename) + if not os.path.exists(tmp_file): + raise exception.NotFound(_('%s not found') % tmp_file) + + @exception.wrap_exception + def cleanup_shared_storage_test_file(self, context, filename): + """Removes existence of the tmpfile under FLAGS.instances_path. + + :param context: security context + :param filename: remove existence of FLAGS.instances_path/thisfile + + """ + + tmp_file = os.path.join(FLAGS.instances_path, filename) + os.remove(tmp_file) @exception.wrap_exception def update_available_resource(self, context): @@ -687,7 +698,7 @@ class ComputeManager(manager.Manager): raise exception.NotFound(msg % locals()) # If any volume is mounted, prepare here. - if len(instance_ref['volumes']) == 0: + if not instance_ref['volumes']: LOG.info(_("%s has no volume."), ec2_id) else: for v in instance_ref['volumes']: @@ -701,16 +712,16 @@ class ComputeManager(manager.Manager): # Retry operation is necessary because continuously request comes, # concorrent request occurs to iptables, then it complains. max_retry = FLAGS.live_migration_retry_count - for i in range(max_retry): + for cnt in range(max_retry): try: self.network_manager.setup_compute_network(context, instance_id) break - except exception.ProcessExecutionError, e: - if i == max_retry - 1: + except exception.ProcessExecutionError: + if cnt == max_retry - 1: raise else: - LOG.warn(_("setup_compute_network() failed %(i)d." + LOG.warn(_("setup_compute_network() failed %(cnt)d." "Retry up to %(max_retry)d for %(ec2_id)s.") % locals()) time.sleep(1) @@ -739,7 +750,7 @@ class ComputeManager(manager.Manager): try: # Checking volume node is working correctly when any volumes # are attached to instances. - if len(instance_ref['volumes']) != 0: + if instance_ref['volumes']: rpc.call(context, FLAGS.volume_topic, {"method": "check_for_export", @@ -751,7 +762,7 @@ class ComputeManager(manager.Manager): {"method": "pre_live_migration", "args": {'instance_id': instance_id}}) - except Exception, e: + except Exception: msg = _("Pre live migration for %(i_name)s failed at %(dest)s") LOG.error(msg % locals()) self.recover_live_migration(context, instance_ref) @@ -843,5 +854,5 @@ class ComputeManager(manager.Manager): 'state': power_state.RUNNING, 'host': host}) - for v in instance_ref['volumes']: - self.db.volume_update(ctxt, v['id'], {'status': 'in-use'}) + for volume in instance_ref['volumes']: + self.db.volume_update(ctxt, volume['id'], {'status': 'in-use'}) diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 73ce651da..4485ba39f 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -100,9 +100,9 @@ class Scheduler(object): 'migrating') # Changing volume state - for v in instance_ref['volumes']: + for volume_ref in instance_ref['volumes']: db.volume_update(context, - v['id'], + volume_ref['id'], {'status': 'migrating'}) # Return value is necessary to send request to src @@ -121,17 +121,16 @@ class Scheduler(object): # Checking instance is running. if (power_state.RUNNING != instance_ref['state'] or \ 'running' != instance_ref['state_description']): - msg = _('Instance(%s) is not running') ec2_id = instance_ref['hostname'] - raise exception.Invalid(msg % ec2_id) + raise exception.Invalid(_('Instance(%s) is not running') % ec2_id) # Checing volume node is running when any volumes are mounted # to the instance. if len(instance_ref['volumes']) != 0: services = db.service_get_all_by_topic(context, 'volume') if len(services) < 1 or not self.service_is_up(services[0]): - msg = _('volume node is not alive(time synchronize problem?)') - raise exception.Invalid(msg) + raise exception.Invalid(_("volume node is not alive" + "(time synchronize problem?)")) # Checking src host exists and compute node src = instance_ref['host'] @@ -139,8 +138,8 @@ class Scheduler(object): # Checking src host is alive. if not self.service_is_up(services[0]): - msg = _('%s is not alive(time synchronize problem?)') - raise exception.Invalid(msg % src) + raise exception.Invalid(_("%s is not alive(time " + "synchronize problem?)") % src) def _live_migration_dest_check(self, context, instance_ref, dest): """Live migration check routine (for destination host). @@ -157,8 +156,8 @@ class Scheduler(object): # Checking dest host is alive. if not self.service_is_up(dservice_ref): - msg = _('%s is not alive(time synchronize problem?)') - raise exception.Invalid(msg % dest) + raise exception.Invalid(_("%s is not alive(time " + "synchronize problem?)") % dest) # Checking whether The host where instance is running # and dest is not same. @@ -170,7 +169,9 @@ class Scheduler(object): % locals()) # Checking dst host still has enough capacities. - self.has_enough_resources(context, instance_ref, dest) + self.assert_compute_node_has_enough_resources(context, + instance_ref, + dest) def _live_migration_common_check(self, context, instance_ref, dest): """Live migration common check routine. @@ -202,18 +203,20 @@ class Scheduler(object): oservice_ref = oservice_refs[0]['compute_service'][0] # Checking hypervisor is same. - o = oservice_ref['hypervisor_type'] - d = dservice_ref['hypervisor_type'] - if o != d: + orig_hypervisor = oservice_ref['hypervisor_type'] + dest_hypervisor = dservice_ref['hypervisor_type'] + if orig_hypervisor != dest_hypervisor: raise exception.Invalid(_("Different hypervisor type" - "(%(o)s->%(d)s)')" % locals())) + "(%(orig_hypervisor)s->" + "%(dest_hypervisor)s)')" % locals())) # Checkng hypervisor version. - o = oservice_ref['hypervisor_version'] - d = dservice_ref['hypervisor_version'] - if o > d: - raise exception.Invalid(_('Older hypervisor version(%(o)s->%(d)s)') - % locals()) + orig_hypervisor = oservice_ref['hypervisor_version'] + dest_hypervisor = dservice_ref['hypervisor_version'] + if orig_hypervisor > dest_hypervisor: + raise exception.Invalid(_("Older hypervisor version" + "(%(orig_hypervisor)s->" + "%(dest_hypervisor)s)") % locals()) # Checking cpuinfo. try: @@ -222,14 +225,15 @@ class Scheduler(object): {"method": 'compare_cpu', "args": {'cpu_info': oservice_ref['cpu_info']}}) - except rpc.RemoteError, e: + except rpc.RemoteError: ec2_id = instance_ref['hostname'] src = instance_ref['host'] logging.exception(_("host %(dest)s is not compatible with " "original host %(src)s.") % locals()) raise - def has_enough_resources(self, context, instance_ref, dest): + def assert_compute_node_has_enough_resources(self, context, + instance_ref, dest): """Checks if destination host has enough resource for live migration. Currently, only memory checking has been done. @@ -276,22 +280,24 @@ class Scheduler(object): dst_t = db.queue_get_for(context, FLAGS.compute_topic, dest) src_t = db.queue_get_for(context, FLAGS.compute_topic, src) - # create tmpfile at dest host try: - filename = rpc.call(context, dst_t, {"method": 'mktmpfile'}) - except rpc.RemoteError, e: - msg = _("Cannot create tmpfile at %s to confirm shared storage.") - LOG.error(msg % FLAGS.instances_path) - raise + # create tmpfile at dest host + filename = rpc.call(context, dst_t, + {"method": 'create_shared_storage_test_file'}) - # make sure existence at src host. - try: + # make sure existence at src host. rpc.call(context, src_t, - {"method": 'confirm_tmpfile', + {"method": 'check_shared_storage_test_file', "args": {'filename': filename}}) - except (rpc.RemoteError, exception.NotFound), e: + except rpc.RemoteError: ipath = FLAGS.instances_path - logging.error(_("Cannot comfirm %(ipath)s at %(dest)s is " - "located at same shared storage.") % locals()) + logging.error(_("Cannot comfirm tmpfile at %(ipath)s is on " + "same shared storage between %(src)s " + "and %(dest)s.") % locals()) raise + + finally: + rpc.call(context, dst_t, + {"method": 'cleanup_shared_storage_test_file', + "args": {'filename': filename}}) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index cd5012fd5..a50d3ab20 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -98,24 +98,24 @@ class SchedulerManager(manager.Manager): # Getting usage resource information usage = {} instance_refs = db.instance_get_all_by_host(context, - compute_ref['host']) - if 0 == len(instance_refs): + compute_ref['host']) + if not instance_refs: return {'resource': resource, 'usage': usage} project_ids = [i['project_id'] for i in instance_refs] project_ids = list(set(project_ids)) - for i in project_ids: + for project_id in project_ids: vcpus = db.instance_get_vcpu_sum_by_host_and_project(context, host, - i) + project_id) mem = db.instance_get_memory_sum_by_host_and_project(context, host, - i) + project_id) hdd = db.instance_get_disk_sum_by_host_and_project(context, host, - i) - usage[i] = {'vcpus': int(vcpus), - 'memory_mb': int(mem), - 'local_gb': int(hdd)} + project_id) + usage[project_id] = {'vcpus': int(vcpus), + 'memory_mb': int(mem), + 'local_gb': int(hdd)} return {'resource': resource, 'usage': usage} diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index c4e4d148e..62db42b11 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -661,7 +661,6 @@ class SimpleDriverTestCase(test.TestCase): self.scheduler.live_migration(self.context, FLAGS.compute_topic, instance_id=instance_id, dest=i_ref['host']) - self.mox.UnsetStubs() i_ref = db.instance_get(self.context, instance_id) self.assertTrue(i_ref['state_description'] == 'migrating') @@ -824,10 +823,15 @@ class SimpleDriverTestCase(test.TestCase): topic = FLAGS.compute_topic driver.rpc.call(mox.IgnoreArg(), db.queue_get_for(self.context, topic, dest), - {"method": 'mktmpfile'}).AndReturn(fpath) + {"method": 'create_shared_storage_test_file'}).AndReturn(fpath) driver.rpc.call(mox.IgnoreArg(), db.queue_get_for(mox.IgnoreArg(), topic, i_ref['host']), - {"method": 'confirm_tmpfile', "args": {'filename': fpath}}) + {"method": 'check_shared_storage_test_file', + "args": {'filename': fpath}}) + driver.rpc.call(mox.IgnoreArg(), + db.queue_get_for(mox.IgnoreArg(), topic, dest), + {"method": 'cleanup_shared_storage_test_file', + "args": {'filename': fpath}}) self.mox.ReplayAll() try: @@ -838,7 +842,6 @@ class SimpleDriverTestCase(test.TestCase): c = (e.message.find('does not exist') >= 0) self.assertTrue(c) - self.mox.UnsetStubs() db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) @@ -867,7 +870,6 @@ class SimpleDriverTestCase(test.TestCase): c = (e.message.find(_('Different hypervisor type')) >= 0) self.assertTrue(c) - self.mox.UnsetStubs() db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) db.service_destroy(self.context, s_ref2['id']) @@ -898,7 +900,6 @@ class SimpleDriverTestCase(test.TestCase): c = (e.message.find(_('Older hypervisor version')) >= 0) self.assertTrue(c) - self.mox.UnsetStubs() db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) db.service_destroy(self.context, s_ref2['id']) @@ -934,7 +935,6 @@ class SimpleDriverTestCase(test.TestCase): c = (e.message.find(_("doesn't have compatibility to")) >= 0) self.assertTrue(c) - self.mox.UnsetStubs() db.instance_destroy(self.context, instance_id) db.service_destroy(self.context, s_ref['id']) db.service_destroy(self.context, s_ref2['id']) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index f46b5950e..17b80c294 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -import libvirt import mox from xml.etree.ElementTree import fromstring as xml_to_tree @@ -60,6 +59,7 @@ class LibvirtConnTestCase(test.TestCase): admin=True) self.project = self.manager.create_project('fake', 'fake', 'fake') self.network = utils.import_object(FLAGS.network_manager) + self.context = context.get_admin_context() FLAGS.instances_path = '' self.call_libvirt_dependant_setup = False @@ -73,22 +73,52 @@ class LibvirtConnTestCase(test.TestCase): 'bridge': 'br101', 'instance_type': 'm1.small'} - def libvirt_dependant_setup(self): - """A setup method of LibvirtConnection dependent test.""" - # try to connect libvirt. if fail, skip test. - self.call_libvirt_dependant_setup = True - try: - libvirt.openReadOnly('qemu:///system') - except libvirt.libvirtError: - return - return libvirt_conn.get_connection(False) - - def libvirt_dependant_teardown(self): - """teardown method of LibvirtConnection dependent test.""" - if self.call_libvirt_dependant_setup: - libvirt_conn.libvirt = None - libvirt_conn.libxml2 = None - self.call_libvirt_dependant_setup = False + def create_fake_libvirt_mock(self, **kwargs): + """Defining mocks for LibvirtConnection(libvirt is not used).""" + + # A fake libvirt.virtConnect + class FakeLibvirtConnection(object): + def getVersion(self): + return 12003 + + def getType(self): + return 'qemu' + + def getCapabilities(self): + return 'qemu' + + def listDomainsID(self): + return [] + + def getCapabilitied(self): + return + + # A fake libvirt_conn.IptablesFirewallDriver + class FakeIptablesFirewallDriver(object): + def __init__(self, **kwargs): + pass + + # Creating mocks + fake = FakeLibvirtConnection() + fakeip = FakeIptablesFirewallDriver + # Customizing above fake if necessary + for key, val in kwargs.items(): + fake.__setattr__(key, val) + + # Inevitable mocks for libvirt_conn.LibvirtConnection + self.mox.StubOutWithMock(libvirt_conn.utils, 'import_class') + libvirt_conn.utils.import_class(mox.IgnoreArg()).AndReturn(fakeip) + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') + libvirt_conn.LibvirtConnection._conn = fake + + def create_service(self, **kwargs): + service_ref = {'host': kwargs.get('host', 'dummy'), + 'binary': 'nova-compute', + 'topic': 'compute', + 'report_count': 0, + 'availability_zone': 'zone'} + + return db.service_create(context.get_admin_context(), service_ref) def test_xml_and_uri_no_ramdisk_no_kernel(self): instance_data = dict(self.test_instance) @@ -244,306 +274,55 @@ class LibvirtConnTestCase(test.TestCase): self.assertEquals(uri, testuri) db.instance_destroy(user_context, instance_ref['id']) - def test_get_vcpu_used(self): - """Check if get_local_gb_total returns appropriate disk value.""" - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') - libvirt_conn.LibvirtConnection._conn.listDomainsID().AndReturn([1, 2]) - vdmock = self.mox.CreateMock(libvirt.virDomain) - self.mox.StubOutWithMock(vdmock, "vcpus") - vdmock.vcpus().AndReturn(['', [('dummycpu'), ('dummycpu')]]) - vdmock.vcpus().AndReturn(['', [('dummycpu'), ('dummycpu')]]) - arg = mox.IgnoreArg() - libvirt_conn.LibvirtConnection._conn.lookupByID(arg).AndReturn(vdmock) - libvirt_conn.LibvirtConnection._conn.lookupByID(arg).AndReturn(vdmock) - - self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) - self.assertTrue(conn.get_vcpu_used() == 4) - - def test_get_cpu_info_inappropreate_xml(self): - """Raise exception if given xml is inappropriate.""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - xml = """ - x86_64 - Nehalem - Intel - - - - - - - - - - - - - - - - - - """ - - self.mox.StubOutWithMock(conn._conn, 'getCapabilities') - conn._conn.getCapabilities().AndReturn(xml) - - self.mox.ReplayAll() - try: - conn.get_cpu_info() - except exception.Invalid, e: - c1 = (0 <= e.message.find('Invalid xml')) - self.assertTrue(c1) - - def test_get_cpu_info_inappropreate_xml2(self): - """Raise exception if given xml is inappropriate(topology tag).""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - xml = """ - x86_64 - Nehalem - Intel - - - - - - - - - - - - - - - - - """ - self.mox.StubOutWithMock(conn._conn, 'getCapabilities') - conn._conn.getCapabilities().AndReturn(xml) - - self.mox.ReplayAll() - try: - conn.get_cpu_info() - except exception.Invalid, e: - c1 = (0 <= e.message.find('Invalid xml: topology')) - self.assertTrue(c1) - def test_update_available_resource_works_correctly(self): """Confirm compute_service table is updated successfully.""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - host = 'dummy' - zone = 'dummyzone' - ctxt = context.get_admin_context() org_path = FLAGS.instances_path = '' FLAGS.instances_path = '.' - service_ref = db.service_create(ctxt, - {'host': host, - 'binary': 'nova-compute', - 'topic': 'compute', - 'report_count': 0, - 'availability_zone': zone}) - conn.update_available_resource(ctxt, host) + service_ref = self.create_service(host='dummy') + self.create_fake_libvirt_mock() + self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, + 'get_cpu_info') + libvirt_conn.LibvirtConnection.get_cpu_info().AndReturn('cpuinfo') - service_ref = db.service_get(ctxt, service_ref['id']) - print service_ref['compute_service'] + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + conn.update_available_resource(self.context, 'dummy') + service_ref = db.service_get(self.context, service_ref['id']) compute_service = service_ref['compute_service'][0] + c1 = (compute_service['vcpus'] > 0) c2 = (compute_service['memory_mb'] > 0) c3 = (compute_service['local_gb'] > 0) - # vcpu_used is checked at test_get_vcpu_used. - c4 = (compute_service['memory_mb_used'] > 0) - c5 = (compute_service['local_gb_used'] > 0) - c6 = (len(compute_service['hypervisor_type']) > 0) - c7 = (compute_service['hypervisor_version'] > 0) + c4 = (compute_service['vcpus_used'] == 0) + c5 = (compute_service['memory_mb_used'] > 0) + c6 = (compute_service['local_gb_used'] > 0) + c7 = (len(compute_service['hypervisor_type']) > 0) + c8 = (compute_service['hypervisor_version'] > 0) - self.assertTrue(c1 and c2 and c3 and c4 and c5 and c6 and c7) + self.assertTrue(c1 and c2 and c3 and c4 and c5 and c6 and c7 and c8) - db.service_destroy(ctxt, service_ref['id']) + db.service_destroy(self.context, service_ref['id']) FLAGS.instances_path = org_path - def test_update_resource_info_raise_exception(self): + def test_update_resource_info_no_compute_record_found(self): """Raise exception if no recorde found on services table.""" - host = 'dummy' - org_path = FLAGS.instances_path = '' - FLAGS.instances_path = '.' - try: - conn = libvirt_conn.LibvirtConnection(False) - conn.update_available_resource(context.get_admin_context(), host) - except exception.Invalid, e: - msg = 'Cannot update compute manager specific info' - c1 = (0 <= e.message.find(msg)) - self.assertTrue(c1) - FLAGS.instances_path = org_path - - def test_compare_cpu_works_correctly(self): - """Calling libvirt.compute_cpu() and works correctly.""" - conn = self.libvirt_dependant_setup() - if not conn: - return - host = 'dummy' - zone = 'dummyzone' - ctxt = context.get_admin_context() org_path = FLAGS.instances_path = '' FLAGS.instances_path = '.' - - service_ref = db.service_create(ctxt, - {'host': host, - 'binary': 'nova-compute', - 'topic': 'compute', - 'report_count': 0, - 'availability_zone': zone}) - conn.update_available_resource(ctxt, host) - service_ref = db.service_get(ctxt, service_ref['id']) - ret = conn.compare_cpu(service_ref['compute_service'][0]['cpu_info']) - self.assertTrue(ret == None) - - db.service_destroy(ctxt, service_ref['id']) - FLAGS.instances_path = org_path - - def test_compare_cpu_no_compatibility(self): - """Libvirt.compare_cpu() return less than 0.(no compatibility).""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - t = {} - t['arch'] = 'x86' - t['model'] = 'model' - t['vendor'] = 'Intel' - t['topology'] = {'cores': "2", "threads": "1", "sockets": "4"} - t['features'] = ["tm"] - cpu_info = utils.dumps(t) - self.mox.StubOutWithMock(conn._conn, 'compareCPU') - conn._conn.compareCPU(mox.IgnoreArg(), 0).AndReturn(0) - - self.mox.ReplayAll() - self.assertRaises(exception.Invalid, conn.compare_cpu, cpu_info) - - def test_ensure_filtering_rules_for_instance_works_correctly(self): - """ensure_filtering_rules_for_instance() works successfully.""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - instance_ref = models.Instance() - instance_ref.__setitem__('id', 1) - fwdriver = conn.firewall_driver - - self.mox.StubOutWithMock(fwdriver, 'setup_basic_filtering') - fwdriver.setup_basic_filtering(instance_ref) - self.mox.StubOutWithMock(fwdriver, 'prepare_instance_filter') - fwdriver.prepare_instance_filter(instance_ref) - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') - n = 'nova-instance-%s' % instance_ref.name - conn._conn.nwfilterLookupByName(n) - - self.mox.ReplayAll() - conn.ensure_filtering_rules_for_instance(instance_ref) - - def test_ensure_filtering_rules_for_instance_timeout(self): - """ensure_filtering_fules_for_instance() finishes with timeout.""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - instance_ref = models.Instance() - instance_ref.__setitem__('id', 1) - fwdriver = conn.firewall_driver - - self.mox.StubOutWithMock(fwdriver, 'setup_basic_filtering') - fwdriver.setup_basic_filtering(instance_ref) - self.mox.StubOutWithMock(fwdriver, 'prepare_instance_filter') - fwdriver.prepare_instance_filter(instance_ref) - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') - n = 'nova-instance-%s' % instance_ref.name - for i in range(FLAGS.live_migration_retry_count): - conn._conn.nwfilterLookupByName(n).\ - AndRaise(libvirt.libvirtError('ERR')) - - self.mox.ReplayAll() - try: - conn.ensure_filtering_rules_for_instance(instance_ref) - except exception.Error, e: - c1 = (0 <= e.message.find('Timeout migrating for')) - self.assertTrue(c1) - - def test_live_migration_works_correctly(self): - """_live_migration() works as expected correctly.""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - class dummyCall(object): - f = None - - def start(self, interval=0, now=False): - pass - - i_ref = models.Instance() - i_ref.__setitem__('id', 1) - ctxt = context.get_admin_context() - - vdmock = self.mox.CreateMock(libvirt.virDomain) - self.mox.StubOutWithMock(vdmock, "migrateToURI") - vdmock.migrateToURI(FLAGS.live_migration_uri % 'dest', - mox.IgnoreArg(), - None, FLAGS.live_migration_bandwidth).\ - AndReturn(None) - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') - conn._conn.lookupByName(i_ref.name).AndReturn(vdmock) - self.mox.StubOutWithMock(libvirt_conn.utils, 'LoopingCall') - libvirt_conn.utils.LoopingCall(f=None).AndReturn(dummyCall()) + self.create_fake_libvirt_mock() self.mox.ReplayAll() - # Nothing to do with setting post_method/recover_method or not. - ret = conn._live_migration(ctxt, i_ref, 'dest', '', '') - self.assertTrue(ret == None) - - def test_live_migration_raises_exception(self): - """Confirms recover method is called when exceptions are raised.""" - conn = self.libvirt_dependant_setup() - if not conn: - return - - i_ref = models.Instance() - i_ref.__setitem__('id', 1) - ctxt = context.get_admin_context() - - def dummy_recover_method(c, instance, host=None): - pass - - vdmock = self.mox.CreateMock(libvirt.virDomain) - self.mox.StubOutWithMock(vdmock, "migrateToURI") - vdmock.migrateToURI(FLAGS.live_migration_uri % 'dest', - mox.IgnoreArg(), - None, FLAGS.live_migration_bandwidth).\ - AndRaise(libvirt.libvirtError('ERR')) - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') - conn._conn.lookupByName(i_ref.name).AndReturn(vdmock) + conn = libvirt_conn.LibvirtConnection(False) + self.assertRaises(exception.Invalid, + conn.update_available_resource, + self.context, 'dummy') - self.mox.ReplayAll() - self.assertRaises(libvirt.libvirtError, - conn._live_migration, - ctxt, i_ref, 'dest', - '', dummy_recover_method) + FLAGS.instances_path = org_path def tearDown(self): self.manager.delete_project(self.project) self.manager.delete_user(self.user) super(LibvirtConnTestCase, self).tearDown() - self.libvirt_dependant_teardown() class IptablesFirewallTestCase(test.TestCase): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 75e4f0a53..70fdcc453 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -891,8 +891,8 @@ class LibvirtConnection(object): """ total = 0 - for i in self._conn.listDomainsID(): - dom = self._conn.lookupByID(i) + for dom_id in self._conn.listDomainsID(): + dom = self._conn.lookupByID(dom_id) total += len(dom.vcpus()[1]) return total @@ -1048,7 +1048,7 @@ class LibvirtConnection(object): 'cpu_info': self.get_cpu_info()} compute_service_ref = service_ref['compute_service'] - if len(compute_service_ref) == 0: + if not compute_service_ref: LOG.info(_('Compute_service record is created for %s ') % host) dic['service_id'] = service_ref['id'] db.compute_service_create(ctxt, dic) @@ -1124,7 +1124,7 @@ class LibvirtConnection(object): # wait for completion timeout_count = range(FLAGS.live_migration_retry_count) - while len(timeout_count) != 0: + while not timeout_count: try: filter_name = 'nova-instance-%s' % instance_ref.name self._conn.nwfilterLookupByName(filter_name) @@ -1198,7 +1198,7 @@ class LibvirtConnection(object): None, FLAGS.live_migration_bandwidth) - except Exception, e: + except Exception: recover_method(ctxt, instance_ref) raise diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 5dc9077b4..9dea35b35 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -175,5 +175,5 @@ class VolumeManager(manager.Manager): def check_for_export(self, context, instance_id): """Make sure whether volume is exported.""" instance_ref = self.db.instance_get(context, instance_id) - for v in instance_ref['volumes']: - self.driver.check_for_export(context, v['id']) + for volume in instance_ref['volumes']: + self.driver.check_for_export(context, volume['id']) -- cgit From 137a4946785b9460aadb9fe40f2b0e18bd7f6063 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Fri, 4 Mar 2011 01:09:21 +0900 Subject: Merged to trunk rev 757. Main changes are below. 1. Rename db table ComputeService -> ComputeNode 2. nova-manage option instance_type is reserved and we cannot use option instance, so change instance -> vm. --- bin/nova-manage | 4 ++-- nova/db/api.py | 12 +++++----- nova/db/sqlalchemy/api.py | 26 +++++++++++----------- .../versions/009_add_live_migration.py | 8 +++---- nova/db/sqlalchemy/models.py | 10 ++++----- nova/scheduler/driver.py | 10 ++++----- nova/scheduler/manager.py | 14 ++++++------ nova/tests/test_scheduler.py | 16 ++++++------- nova/tests/test_virt.py | 22 +++++++++--------- nova/virt/libvirt_conn.py | 10 ++++----- 10 files changed, 66 insertions(+), 66 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index f41950cd2..d782f6028 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -546,7 +546,7 @@ class NetworkCommands(object): network.dns) -class InstanceCommands(object): +class VmCommands(object): """Class for mangaging VM instances.""" def live_migration(self, ec2_id, dest): @@ -831,7 +831,7 @@ CATEGORIES = [ ('fixed', FixedIpCommands), ('floating', FloatingIpCommands), ('network', NetworkCommands), - ('instance', InstanceCommands), + ('vm', VmCommands), ('service', ServiceCommands), ('log', LogCommands), ('db', DbCommands), diff --git a/nova/db/api.py b/nova/db/api.py index 13bc07ad2..3b427cefe 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -153,24 +153,24 @@ def service_update(context, service_id, values): ################### -def compute_service_get(context, compute_id, session=None): +def compute_node_get(context, compute_id, session=None): """Get an computeService or raise if it does not exist.""" - return IMPL.compute_service_get(context, compute_id) + return IMPL.compute_node_get(context, compute_id) -def compute_service_create(context, values): +def compute_node_create(context, values): """Create a computeService from the values dictionary.""" - return IMPL.compute_service_create(context, values) + return IMPL.compute_node_create(context, values) -def compute_service_update(context, compute_id, values): +def compute_node_update(context, compute_id, values): """Set the given properties on an computeService and update it. Raises NotFound if computeService does not exist. """ - return IMPL.compute_service_update(context, compute_id, values) + return IMPL.compute_node_update(context, compute_id, values) ################### diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index bed621b18..69aa07279 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -119,8 +119,8 @@ def service_destroy(context, service_id): service_ref.delete(session=session) if service_ref.topic == 'compute' and \ - len(service_ref.compute_service) != 0: - for c in service_ref.compute_service: + len(service_ref.compute_node) != 0: + for c in service_ref.compute_node: c.delete(session=session) @@ -130,7 +130,7 @@ def service_get(context, service_id, session=None): session = get_session() result = session.query(models.Service).\ - options(joinedload('compute_service')).\ + options(joinedload('compute_node')).\ filter_by(id=service_id).\ filter_by(deleted=can_read_deleted(context)).\ first() @@ -174,7 +174,7 @@ def service_get_all_compute_by_host(context, host): topic = 'compute' session = get_session() result = session.query(models.Service).\ - options(joinedload('compute_service')).\ + options(joinedload('compute_node')).\ filter_by(deleted=False).\ filter_by(host=host).\ filter_by(topic=topic).\ @@ -298,11 +298,11 @@ def service_update(context, service_id, values): @require_admin_context -def compute_service_get(context, compute_id, session=None): +def compute_node_get(context, compute_id, session=None): if not session: session = get_session() - result = session.query(models.ComputeService).\ + result = session.query(models.ComputeNode).\ filter_by(id=compute_id).\ filter_by(deleted=can_read_deleted(context)).\ first() @@ -314,18 +314,18 @@ def compute_service_get(context, compute_id, session=None): @require_admin_context -def compute_service_create(context, values): - compute_service_ref = models.ComputeService() - compute_service_ref.update(values) - compute_service_ref.save() - return compute_service_ref +def compute_node_create(context, values): + compute_node_ref = models.ComputeNode() + compute_node_ref.update(values) + compute_node_ref.save() + return compute_node_ref @require_admin_context -def compute_service_update(context, compute_id, values): +def compute_node_update(context, compute_id, values): session = get_session() with session.begin(): - compute_ref = compute_service_get(context, compute_id, session=session) + compute_ref = compute_node_get(context, compute_id, session=session) compute_ref.update(values) compute_ref.save(session=session) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/009_add_live_migration.py b/nova/db/sqlalchemy/migrate_repo/versions/009_add_live_migration.py index 2689b5b74..23ccccb4e 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/009_add_live_migration.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/009_add_live_migration.py @@ -31,7 +31,7 @@ instances = Table('instances', meta, # New Tables # -compute_services = Table('compute_services', meta, +compute_nodes = Table('compute_nodes', meta, Column('created_at', DateTime(timezone=False)), Column('updated_at', DateTime(timezone=False)), Column('deleted_at', DateTime(timezone=False)), @@ -73,11 +73,11 @@ def upgrade(migrate_engine): meta.bind = migrate_engine try: - compute_services.create() + compute_nodes.create() except Exception: - logging.info(repr(compute_services)) + logging.info(repr(compute_nodes)) logging.exception('Exception while creating table') - meta.drop_all(tables=[compute_services]) + meta.drop_all(tables=[compute_nodes]) raise instances.create_column(instances_launched_on) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 2af7377ef..8646190f3 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -113,18 +113,18 @@ class Service(BASE, NovaBase): availability_zone = Column(String(255), default='nova') -class ComputeService(BASE, NovaBase): +class ComputeNode(BASE, NovaBase): """Represents a running compute service on a host.""" - __tablename__ = 'compute_services' + __tablename__ = 'compute_nodes' id = Column(Integer, primary_key=True) service_id = Column(Integer, ForeignKey('services.id'), nullable=True) service = relationship(Service, - backref=backref('compute_service'), + backref=backref('compute_node'), foreign_keys=service_id, primaryjoin='and_(' - 'ComputeService.service_id == Service.id,' - 'ComputeService.deleted == False)') + 'ComputeNode.service_id == Service.id,' + 'ComputeNode.deleted == False)') vcpus = Column(Integer, nullable=True) memory_mb = Column(Integer, nullable=True) diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 4485ba39f..791f9000d 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -190,7 +190,7 @@ class Scheduler(object): # Checking dest exists. dservice_refs = db.service_get_all_compute_by_host(context, dest) - dservice_ref = dservice_refs[0]['compute_service'][0] + dservice_ref = dservice_refs[0]['compute_node'][0] # Checking original host( where instance was launched at) exists. try: @@ -200,7 +200,7 @@ class Scheduler(object): raise exception.Invalid(_("host %s where instance was launched " "does not exist.") % instance_ref['launched_on']) - oservice_ref = oservice_refs[0]['compute_service'][0] + oservice_ref = oservice_refs[0]['compute_node'][0] # Checking hypervisor is same. orig_hypervisor = oservice_ref['hypervisor_type'] @@ -252,10 +252,10 @@ class Scheduler(object): # Getting host information service_refs = db.service_get_all_compute_by_host(context, dest) - compute_service_ref = service_refs[0]['compute_service'][0] + compute_node_ref = service_refs[0]['compute_node'][0] - mem_total = int(compute_service_ref['memory_mb']) - mem_used = int(compute_service_ref['memory_mb_used']) + mem_total = int(compute_node_ref['memory_mb']) + mem_used = int(compute_node_ref['memory_mb_used']) mem_avail = mem_total - mem_used mem_inst = instance_ref['memory_mb'] if mem_avail <= mem_inst: diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index a50d3ab20..090d8b89d 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -87,13 +87,13 @@ class SchedulerManager(manager.Manager): compute_ref = compute_ref[0] # Getting physical resource information - compute_service_ref = compute_ref['compute_service'][0] - resource = {'vcpus': compute_service_ref['vcpus'], - 'memory_mb': compute_service_ref['memory_mb'], - 'local_gb': compute_service_ref['local_gb'], - 'vcpus_used': compute_service_ref['vcpus_used'], - 'memory_mb_used': compute_service_ref['memory_mb_used'], - 'local_gb_used': compute_service_ref['local_gb_used']} + compute_node_ref = compute_ref['compute_node'][0] + resource = {'vcpus': compute_node_ref['vcpus'], + 'memory_mb': compute_node_ref['memory_mb'], + 'local_gb': compute_node_ref['local_gb'], + 'vcpus_used': compute_node_ref['vcpus_used'], + 'memory_mb_used': compute_node_ref['memory_mb_used'], + 'local_gb_used': compute_node_ref['local_gb_used']} # Getting usage resource information usage = {} diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 62db42b11..711b66af7 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -60,7 +60,7 @@ class SchedulerTestCase(test.TestCase): self.flags(scheduler_driver='nova.tests.test_scheduler.TestDriver') def _create_compute_service(self): - """Create compute-manager(ComputeService and Service record).""" + """Create compute-manager(ComputeNode and Service record).""" ctxt = context.get_admin_context() dic = {'host': 'dummy', 'binary': 'nova-compute', 'topic': 'compute', 'report_count': 0, 'availability_zone': 'dummyzone'} @@ -71,7 +71,7 @@ class SchedulerTestCase(test.TestCase): 'vcpus_used': 16, 'memory_mb_used': 32, 'local_gb_used': 10, 'hypervisor_type': 'qemu', 'hypervisor_version': 12003, 'cpu_info': ''} - db.compute_service_create(ctxt, dic) + db.compute_node_create(ctxt, dic) return db.service_get(ctxt, s_ref['id']) @@ -144,8 +144,8 @@ class SchedulerTestCase(test.TestCase): # result checking c1 = ('resource' in result and 'usage' in result) - compute_service = s_ref['compute_service'][0] - c2 = self._dic_is_equal(result['resource'], compute_service) + compute_node = s_ref['compute_node'][0] + c2 = self._dic_is_equal(result['resource'], compute_node) c3 = result['usage'] == {} self.assertTrue(c1 and c2 and c3) db.service_destroy(ctxt, s_ref['id']) @@ -163,8 +163,8 @@ class SchedulerTestCase(test.TestCase): result = scheduler.show_host_resources(ctxt, s_ref['host']) c1 = ('resource' in result and 'usage' in result) - compute_service = s_ref['compute_service'][0] - c2 = self._dic_is_equal(result['resource'], compute_service) + compute_node = s_ref['compute_node'][0] + c2 = self._dic_is_equal(result['resource'], compute_node) c3 = result['usage'].keys() == ['p-01', 'p-02'] keys = ['vcpus', 'memory_mb', 'local_gb'] c4 = self._dic_is_equal(result['usage']['p-01'], i_ref1, keys) @@ -301,7 +301,7 @@ class SimpleDriverTestCase(test.TestCase): dic['memory_mb_used'] = kwargs.get('memory_mb_used', 32) dic['hypervisor_type'] = kwargs.get('hypervisor_type', 'qemu') dic['hypervisor_version'] = kwargs.get('hypervisor_version', 12003) - db.compute_service_create(self.context, dic) + db.compute_node_create(self.context, dic) return db.service_get(self.context, s_ref['id']) def test_doesnt_report_disabled_hosts_as_up(self): @@ -923,7 +923,7 @@ class SimpleDriverTestCase(test.TestCase): self.mox.StubOutWithMock(rpc, 'call', use_mock_anything=True) rpc.call(mox.IgnoreArg(), mox.IgnoreArg(), {"method": 'compare_cpu', - "args": {'cpu_info': s_ref2['compute_service'][0]['cpu_info']}}).\ + "args": {'cpu_info': s_ref2['compute_node'][0]['cpu_info']}}).\ AndRaise(rpc.RemoteError("doesn't have compatibility to", "", "")) self.mox.ReplayAll() diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 17b80c294..aac55a894 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -275,7 +275,7 @@ class LibvirtConnTestCase(test.TestCase): db.instance_destroy(user_context, instance_ref['id']) def test_update_available_resource_works_correctly(self): - """Confirm compute_service table is updated successfully.""" + """Confirm compute_node table is updated successfully.""" org_path = FLAGS.instances_path = '' FLAGS.instances_path = '.' @@ -289,16 +289,16 @@ class LibvirtConnTestCase(test.TestCase): conn = libvirt_conn.LibvirtConnection(False) conn.update_available_resource(self.context, 'dummy') service_ref = db.service_get(self.context, service_ref['id']) - compute_service = service_ref['compute_service'][0] - - c1 = (compute_service['vcpus'] > 0) - c2 = (compute_service['memory_mb'] > 0) - c3 = (compute_service['local_gb'] > 0) - c4 = (compute_service['vcpus_used'] == 0) - c5 = (compute_service['memory_mb_used'] > 0) - c6 = (compute_service['local_gb_used'] > 0) - c7 = (len(compute_service['hypervisor_type']) > 0) - c8 = (compute_service['hypervisor_version'] > 0) + compute_node = service_ref['compute_node'][0] + + c1 = (compute_node['vcpus'] > 0) + c2 = (compute_node['memory_mb'] > 0) + c3 = (compute_node['local_gb'] > 0) + c4 = (compute_node['vcpus_used'] == 0) + c5 = (compute_node['memory_mb_used'] > 0) + c6 = (compute_node['local_gb_used'] > 0) + c7 = (len(compute_node['hypervisor_type']) > 0) + c8 = (compute_node['hypervisor_version'] > 0) self.assertTrue(c1 and c2 and c3 and c4 and c5 and c6 and c7 and c8) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index b9abf1890..71ca508b0 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1021,7 +1021,7 @@ class LibvirtConnection(object): self.firewall_driver.refresh_security_group_members(security_group_id) def update_available_resource(self, ctxt, host): - """Updates compute manager resource info on ComputeService table. + """Updates compute manager resource info on ComputeNode table. This method is called when nova-coompute launches, and whenever admin executes "nova-manage service update_resource". @@ -1049,14 +1049,14 @@ class LibvirtConnection(object): 'hypervisor_version': self.get_hypervisor_version(), 'cpu_info': self.get_cpu_info()} - compute_service_ref = service_ref['compute_service'] - if not compute_service_ref: + compute_node_ref = service_ref['compute_node'] + if not compute_node_ref: LOG.info(_('Compute_service record is created for %s ') % host) dic['service_id'] = service_ref['id'] - db.compute_service_create(ctxt, dic) + db.compute_node_create(ctxt, dic) else: LOG.info(_('Compute_service record is updated for %s ') % host) - db.compute_service_update(ctxt, compute_service_ref[0]['id'], dic) + db.compute_node_update(ctxt, compute_node_ref[0]['id'], dic) def compare_cpu(self, cpu_info): """Checks the host cpu is compatible to a cpu given by xml. -- cgit From bc6cc457132b096150dcd9ff2ed2909585a80484 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Fri, 4 Mar 2011 01:17:05 +0900 Subject: some comments are modified --- nova/db/api.py | 8 ++++---- nova/db/sqlalchemy/api.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nova/db/api.py b/nova/db/api.py index 3b427cefe..43e1c2183 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -154,19 +154,19 @@ def service_update(context, service_id, values): def compute_node_get(context, compute_id, session=None): - """Get an computeService or raise if it does not exist.""" + """Get an computeNode or raise if it does not exist.""" return IMPL.compute_node_get(context, compute_id) def compute_node_create(context, values): - """Create a computeService from the values dictionary.""" + """Create a computeNode from the values dictionary.""" return IMPL.compute_node_create(context, values) def compute_node_update(context, compute_id, values): - """Set the given properties on an computeService and update it. + """Set the given properties on an computeNode and update it. - Raises NotFound if computeService does not exist. + Raises NotFound if computeNode does not exist. """ diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 69aa07279..b305543ff 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -308,7 +308,7 @@ def compute_node_get(context, compute_id, session=None): first() if not result: - raise exception.NotFound(_('No computeService for id %s') % compute_id) + raise exception.NotFound(_('No computeNode for id %s') % compute_id) return result -- cgit From 9cfe8ff2e8e66952c3202b852a88ee6fca6fb736 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 3 Mar 2011 16:31:01 -0500 Subject: pep8 --- nova/api/openstack/servers.py | 9 +++++---- nova/tests/api/openstack/test_servers.py | 6 +++--- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 73c787828..ea13116fa 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -85,6 +85,7 @@ def _translate_detail_keys(inst): return dict(server=inst_dict) + def _translate_keys(inst): """ Coerces into dictionary format, excluding all model attributes save for id and name """ @@ -143,8 +144,8 @@ class Controller(wsgi.Controller): """ Create a list of onset files from the personality request attribute - At this time, onset_files must be formatted as a list of - (file_path, file_content) pairs for compatibility with the + At this time, onset_files must be formatted as a list of + (file_path, file_content) pairs for compatibility with the underlying compute service. """ onset_files = [] @@ -157,8 +158,8 @@ class Controller(wsgi.Controller): try: contents = base64.b64decode(contents) except TypeError: - raise exc.HTTPBadRequest(explanation= - 'Personality content for %s cannot be decoded' % path) + msg = 'Personality content for %s cannot be decoded' % path + raise exc.HTTPBadRequest(explanation=msg) onset_files.append((path, contents)) return onset_files diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 272d34e3a..53cfa3a6e 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -287,8 +287,8 @@ class ServersTest(test.TestCase): request, response, onset_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [(path, contents)]) - + self.assertEquals(onset_files, [(path, contents)]) + def test_create_instance_with_personality_with_non_b64_content(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Oh no!"\n' @@ -320,7 +320,7 @@ class ServersTest(test.TestCase): request, response, onset_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [(path, contents)]) + self.assertEquals(onset_files, [(path, contents)]) def test_create_instance_personality_not_a_list(self): path = '/my/file/path' -- cgit From e14f524eb92ae07704a2ec7dac0f97c60940a6ab Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 3 Mar 2011 17:49:41 -0500 Subject: enforce personality quotas --- nova/compute/api.py | 24 ++++++++++++++++++ nova/quota.py | 27 ++++++++++++-------- nova/tests/test_quota.py | 66 +++++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 101 insertions(+), 16 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index 625778b66..44e583cd4 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -80,6 +80,26 @@ class API(base.Base): topic, {"method": "get_network_topic", "args": {'fake': 1}}) + def _check_personality_file_quota(self, context, personality_files): + limit = quota.allowed_personality_files(context) + if len(personality_files) > limit: + raise quota.QuotaError(_("Personality limit exceeded. You can " + "only have %d personalities when " + "creating an instance.") % limit, + "PersonalityLimitExceeded") + path_limit = quota.allowed_personality_path_bytes(context) + content_limit = quota.allowed_personality_content_bytes(context) + for path, content in personality_files: + if len(path) > path_limit: + raise quota.QuotaError( + _("Personality file path limit exceeded."), + "PersonalityLimitExceeded") + if len(content) > content_limit: + raise quota.QuotaError( + _("Personality file content limit exceeded."), + "PersonalityLimitExceeded") + return personality_files + def create(self, context, instance_type, image_id, kernel_id=None, ramdisk_id=None, min_count=1, max_count=1, @@ -124,6 +144,10 @@ class API(base.Base): LOG.warn(msg) raise quota.QuotaError(msg, "MetadataLimitExceeded") + if onset_files is not None: + onset_files = \ + self._check_personality_file_quota(context, onset_files) + image = self.image_service.show(context, image_id) if kernel_id is None: kernel_id = image.get('kernel_id', None) diff --git a/nova/quota.py b/nova/quota.py index 14b388794..4b777624c 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -37,10 +37,12 @@ flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') flags.DEFINE_integer('quota_metadata_items', 128, 'number of metadata items allowed per instance') -flags.DEFINE_integer('quota_file_injection_max_files', 5, - 'number of files allowed during file injection') -flags.DEFINE_integer('quota_file_injection_max_file_bytes', 10 * 1024, - 'number of bytes allowed per file during file injection') +flags.DEFINE_integer('quota_personality_max_files', 5, + 'number of personality files allowed') +flags.DEFINE_integer('quota_personality_max_content_bytes', 10 * 1024, + 'number of bytes allowed per personality file') +flags.DEFINE_integer('quota_personality_max_path_bytes', 255, + 'number of bytes allowed per personality file path') def get_quota(context, project_id): @@ -113,14 +115,19 @@ def allowed_metadata_items(context, num_metadata_items): return min(num_metadata_items, num_allowed_metadata_items) -def allowed_file_injection_files(context): - """Return the number of files allowed per file injection""" - return FLAGS.quota_file_injection_max_files +def allowed_personality_files(context): + """Return the number of personality files allowed""" + return FLAGS.quota_personality_max_files -def allowed_file_injection_file_bytes(context): - """Return the number of bytes allowed per file during injection""" - return FLAGS.quota_file_injection_max_file_bytes +def allowed_personality_content_bytes(context): + """Return the number of bytes allowed per personality content""" + return FLAGS.quota_personality_max_content_bytes + + +def allowed_personality_path_bytes(context): + """Return the number of bytes allowed in a personality file path""" + return FLAGS.quota_personality_max_path_bytes class QuotaError(exception.ApiError): diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 48e5a5538..16a083788 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -177,12 +177,66 @@ class QuotaTestCase(test.TestCase): image_id='fake', metadata=metadata) - def test_allowed_file_injection_files(self): + def test_allowed_personality_files(self): self.assertEqual( - quota.allowed_file_injection_files(self.context), - FLAGS.quota_file_injection_max_files) + quota.allowed_personality_files(self.context), + FLAGS.quota_personality_max_files) + + def _create_with_personality(self, files): + api = compute.API() + api.create(self.context, min_count=1, max_count=1, + instance_type='m1.small', image_id='fake', + onset_files=files) + + def test_no_personality_files(self): + api = compute.API() + api.create(self.context, instance_type='m1.small', image_id='fake') + + def test_max_personality_files(self): + files = [] + for i in xrange(FLAGS.quota_personality_max_files): + files.append(('/my/path%d' % i, 'config = test\n')) + self._create_with_personality(files) # no QuotaError + + def test_too_many_personality_files(self): + files = [] + for i in xrange(FLAGS.quota_personality_max_files + 1): + files.append(('/my/path%d' % i, 'my\ncontent%d\n' % i)) + self.assertRaises(quota.QuotaError, + self._create_with_personality, files) - def test_allowed_file_injection_file_bytes(self): + def test_allowed_personality_content_bytes(self): self.assertEqual( - quota.allowed_file_injection_file_bytes(self.context), - FLAGS.quota_file_injection_max_file_bytes) + quota.allowed_personality_content_bytes(self.context), + FLAGS.quota_personality_max_content_bytes) + + def test_max_personality_content_bytes(self): + max = FLAGS.quota_personality_max_content_bytes + content = ''.join(['a' for i in xrange(max)]) + files = [('/test/path', content)] + self._create_with_personality(files) # no QuotaError + + def test_too_many_personality_content_bytes(self): + max = FLAGS.quota_personality_max_content_bytes + content = ''.join(['a' for i in xrange(max + 1)]) + files = [('/test/path', content)] + self.assertRaises(quota.QuotaError, + self._create_with_personality, files) + + def test_allowed_personality_path_bytes(self): + self.assertEqual( + quota.allowed_personality_path_bytes(self.context), + FLAGS.quota_personality_max_path_bytes) + + def test_max_personality_path_bytes(self): + max = FLAGS.quota_personality_max_path_bytes + path = ''.join(['a' for i in xrange(max)]) + files = [(path, 'config = quotatest')] + self._create_with_personality(files) # no QuotaError + + def test_too_many_personality_path_bytes(self): + max = FLAGS.quota_personality_max_path_bytes + path = ''.join(['a' for i in xrange(max + 1)]) + files = [(path, 'config = quotatest')] + self.assertRaises(quota.QuotaError, + self._create_with_personality, files) -- cgit From 5ae13551990be67e3509ddcd10d1872a91634d83 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 3 Mar 2011 18:27:57 -0500 Subject: rename onset_files to personality_files all the way down to compute manager --- nova/api/openstack/servers.py | 16 +++++++-------- nova/compute/api.py | 10 ++++----- nova/compute/manager.py | 2 +- nova/tests/api/openstack/test_servers.py | 35 ++++++++++++++++---------------- nova/tests/test_quota.py | 2 +- 5 files changed, 33 insertions(+), 32 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index ea13116fa..8f6d8de66 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -140,15 +140,15 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def _get_onset_files_from_personality(self, personality): + def _get_personality_files(self, personality): """ - Create a list of onset files from the personality request attribute + Create a list of personality files from the personality attribute - At this time, onset_files must be formatted as a list of + At this time, personality_files must be formatted as a list of (file_path, file_content) pairs for compatibility with the underlying compute service. """ - onset_files = [] + personality_files = [] for item in personality: try: path = item['path'] @@ -160,8 +160,8 @@ class Controller(wsgi.Controller): except TypeError: msg = 'Personality content for %s cannot be decoded' % path raise exc.HTTPBadRequest(explanation=msg) - onset_files.append((path, contents)) - return onset_files + personality_files.append((path, contents)) + return personality_files def create(self, req): """ Creates a new server for a given user """ @@ -191,7 +191,7 @@ class Controller(wsgi.Controller): metadata.append({'key': k, 'value': v}) personality = env['server'].get('personality', []) - onset_files = self._get_onset_files_from_personality(personality) + personality_files = self._get_personality_files(personality) instances = self.compute_api.create( context, @@ -204,7 +204,7 @@ class Controller(wsgi.Controller): key_name=key_pair['name'], key_data=key_pair['public_key'], metadata=metadata, - onset_files=onset_files) + personality_files=personality_files) return _translate_keys(instances[0]) def update(self, req, id): diff --git a/nova/compute/api.py b/nova/compute/api.py index 44e583cd4..13938dcde 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -106,7 +106,7 @@ class API(base.Base): display_name='', display_description='', key_name=None, key_data=None, security_group='default', availability_zone=None, user_data=None, metadata=[], - onset_files=None): + personality_files=None): """Create the number of instances requested if quota and other arguments check out ok. """ @@ -144,9 +144,9 @@ class API(base.Base): LOG.warn(msg) raise quota.QuotaError(msg, "MetadataLimitExceeded") - if onset_files is not None: - onset_files = \ - self._check_personality_file_quota(context, onset_files) + if personality_files is not None: + personality_files = \ + self._check_personality_file_quota(context, personality_files) image = self.image_service.show(context, image_id) if kernel_id is None: @@ -242,7 +242,7 @@ class API(base.Base): "args": {"topic": FLAGS.compute_topic, "instance_id": instance_id, "availability_zone": availability_zone, - "onset_files": onset_files}}) + "personality_files": personality_files}}) for group_id in security_groups: self.trigger_security_group_members_refresh(elevated, group_id) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index d659712ad..1a392dda8 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -174,7 +174,7 @@ class ComputeManager(manager.Manager): """Launch a new instance with specified options.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) - instance_ref.onset_files = kwargs.get('onset_files', []) + instance_ref.onset_files = kwargs.get('personality_files', []) if instance_ref['name'] in self.driver.list_instances(): raise exception.Error(_("Instance has already been created")) LOG.audit(_("instance %s: starting..."), instance_id, diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 53cfa3a6e..bf934113a 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -241,13 +241,13 @@ class ServersTest(test.TestCase): class FakeComputeAPI(object): def __init__(self): - self.onset_files = None + self.personality_files = None def create(self, *args, **kwargs): - if 'onset_files' in kwargs: - self.onset_files = kwargs['onset_files'] + if 'personality_files' in kwargs: + self.personality_files = kwargs['personality_files'] else: - self.onset_files = None + self.personality_files = None return [{'id': '1234', 'display_name': 'fakeinstance'}] def make_stub_method(canned_return): @@ -271,32 +271,33 @@ class ServersTest(test.TestCase): req = webob.Request.blank('/v1.0/servers') req.method = 'POST' req.body = json.dumps(body) - return req, req.get_response(fakes.wsgi_app()), compute_api.onset_files + return (req, req.get_response(fakes.wsgi_app()), + compute_api.personality_files) def test_create_instance_with_no_personality(self): - request, response, onset_files = \ + request, response, personality_files = \ self._create_instance_with_personality(personality=None) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, []) + self.assertEquals(personality_files, []) def test_create_instance_with_personality(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Hello, World!"\n' b64contents = base64.b64encode(contents) personality = [self._personality_dict(path, b64contents)] - request, response, onset_files = \ + request, response, personality_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [(path, contents)]) + self.assertEquals(personality_files, [(path, contents)]) def test_create_instance_with_personality_with_non_b64_content(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Oh no!"\n' personality = [self._personality_dict(path, contents)] - request, response, onset_files = \ + request, response, personality_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 400) - self.assertEquals(onset_files, None) + self.assertEquals(personality_files, None) def test_create_instance_with_three_personalities(self): files = [ @@ -308,28 +309,28 @@ class ServersTest(test.TestCase): for path, content in files: personality.append(self._personality_dict( path, base64.b64encode(content))) - request, response, onset_files = \ + request, response, personality_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, files) + self.assertEquals(personality_files, files) def test_create_instance_personality_empty_content(self): path = '/my/file/path' contents = '' personality = [self._personality_dict(path, contents)] - request, response, onset_files = \ + request, response, personality_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [(path, contents)]) + self.assertEquals(personality_files, [(path, contents)]) def test_create_instance_personality_not_a_list(self): path = '/my/file/path' contents = 'myfilecontents' personality = self._personality_dict(path, contents) - request, response, onset_files = \ + request, response, personality_files = \ self._create_instance_with_personality(personality) self.assertEquals(response.status_int, 400) - self.assertEquals(onset_files, None) + self.assertEquals(personality_files, None) def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 16a083788..b26dec61a 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -186,7 +186,7 @@ class QuotaTestCase(test.TestCase): api = compute.API() api.create(self.context, min_count=1, max_count=1, instance_type='m1.small', image_id='fake', - onset_files=files) + personality_files=files) def test_no_personality_files(self): api = compute.API() -- cgit From abd5779068f3b979fc79dec7a68549999c58092d Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Fri, 4 Mar 2011 01:36:29 -0500 Subject: remove ensure_b64_encoding --- nova/compute/manager.py | 10 ++-------- nova/utils.py | 12 ------------ nova/virt/xenapi/vmops.py | 10 +++++----- 3 files changed, 7 insertions(+), 25 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index d915dc069..3a712fd97 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -34,7 +34,6 @@ terminating it. :func:`nova.utils.import_object` """ -import base64 import datetime import random import string @@ -353,15 +352,10 @@ class ComputeManager(manager.Manager): LOG.warn(_('trying to inject a file into a non-running ' 'instance: %(instance_id)s (state: %(instance_state)s ' 'expected: %(expected_state)s)') % locals()) - # Files/paths *should* be base64-encoded at this point, but - # double-check to make sure. - b64_path = utils.ensure_b64_encoding(path) - b64_contents = utils.ensure_b64_encoding(file_contents) - plain_path = base64.b64decode(b64_path) nm = instance_ref['name'] - msg = _('instance %(nm)s: injecting file to %(plain_path)s') % locals() + msg = _('instance %(nm)s: injecting file to %(path)s') % locals() LOG.audit(msg) - self.driver.inject_file(instance_ref, b64_path, b64_contents) + self.driver.inject_file(instance_ref, path, file_contents) @exception.wrap_exception @checks_instance_lock diff --git a/nova/utils.py b/nova/utils.py index 0cf91e0cc..02b71900c 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -491,18 +491,6 @@ def loads(s): return json.loads(s) -def ensure_b64_encoding(val): - """Safety method to ensure that values expected to be base64-encoded - actually are. If they are, the value is returned unchanged. Otherwise, - the encoded value is returned. - """ - try: - dummy = base64.decode(val) - return val - except TypeError: - return base64.b64encode(val) - - def get_from_path(items, path): """ Returns a list of items matching the specified path. Takes an XPath-like expression e.g. prop1/prop2/prop3, and for each item in items, diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 9ac83efb0..89d58a664 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -19,6 +19,7 @@ Management class for VM-related functions (spawn, reboot, etc). """ +import base64 import json import M2Crypto import os @@ -313,17 +314,16 @@ class VMOps(object): task = self._session.call_xenapi("Async.VM.start", vm, False, False) self._session.wait_for_task(task, instance.id) - def inject_file(self, instance, b64_path, b64_contents): + def inject_file(self, instance, path, contents): """Write a file to the VM instance. The path to which it is to be written and the contents of the file need to be supplied; both should be base64-encoded to prevent errors with non-ASCII characters being transmitted. If the agent does not support file injection, or the user has disabled it, a NotImplementedError will be raised. """ - # Files/paths *should* be base64-encoded at this point, but - # double-check to make sure. - b64_path = utils.ensure_b64_encoding(b64_path) - b64_contents = utils.ensure_b64_encoding(b64_contents) + # Files/paths must be base64-encoded for transmission to agent + b64_path = base64.b64encode(path) + b64_contents = base64.b64encode(contents) # Need to uniquely identify this request. transaction_id = str(uuid.uuid4()) -- cgit From 1f0df07baac52379b122a9928200305dd9d2151f Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Sat, 5 Mar 2011 00:57:08 +0900 Subject: Fixed based on reviewer's comment. Main changes are below. 1. get_vcpu_total()/get_memory_mb()/get_memory_mb_used() is changed for users who used non-linux environment. 2. test code added to test_virt. --- contrib/nova.sh | 1 + nova/tests/test_virt.py | 163 ++++++++++++++++++++++++++++++++++++++-------- nova/virt/libvirt_conn.py | 12 +++- 3 files changed, 147 insertions(+), 29 deletions(-) diff --git a/contrib/nova.sh b/contrib/nova.sh index 1187f2728..cf5b3de11 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -76,6 +76,7 @@ if [ "$CMD" == "install" ]; then sudo apt-get install -y python-migrate python-eventlet python-gflags python-ipy python-tempita sudo apt-get install -y python-libvirt python-libxml2 python-routes python-cheetah sudo apt-get install -y python-netaddr python-paste python-pastedeploy python-glance + sudo apt-get install -y python-multiprocessing if [ "$USE_IPV6" == 1 ]; then sudo apt-get install -y radvd diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index aac55a894..5bb31659b 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -15,6 +15,7 @@ # under the License. import mox +import sys from xml.etree.ElementTree import fromstring as xml_to_tree from xml.dom.minidom import parseString as xml_to_dom @@ -27,11 +28,15 @@ from nova import test from nova import utils from nova.api.ec2 import cloud from nova.auth import manager +from nova.compute import manager as compute_manager +from nova.compute import power_state from nova.db.sqlalchemy import models from nova.virt import libvirt_conn +libvirt = None FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') +flags.DECLARE('compute_driver', 'nova.compute.manager') class LibvirtConnTestCase(test.TestCase): @@ -73,31 +78,36 @@ class LibvirtConnTestCase(test.TestCase): 'bridge': 'br101', 'instance_type': 'm1.small'} + def lazy_load_library_exists(self): + """check if libvirt is available.""" + # try to connect libvirt. if fail, skip test. + try: + import libvirt + import libxml2 + except ImportError: + return False + global libvirt + libvirt = __import__('libvirt') + libvirt_conn.libvirt = __import__('libvirt') + libvirt_conn.libxml2 = __import__('libxml2') + return True + def create_fake_libvirt_mock(self, **kwargs): """Defining mocks for LibvirtConnection(libvirt is not used).""" - # A fake libvirt.virtConnect + # A fake libvirt.virConnect class FakeLibvirtConnection(object): - def getVersion(self): - return 12003 - - def getType(self): - return 'qemu' - - def getCapabilities(self): - return 'qemu' - - def listDomainsID(self): - return [] - - def getCapabilitied(self): - return + pass # A fake libvirt_conn.IptablesFirewallDriver class FakeIptablesFirewallDriver(object): + def __init__(self, **kwargs): pass + def setattr(self, key, val): + self.__setattr__(key, val) + # Creating mocks fake = FakeLibvirtConnection() fakeip = FakeIptablesFirewallDriver @@ -274,33 +284,54 @@ class LibvirtConnTestCase(test.TestCase): self.assertEquals(uri, testuri) db.instance_destroy(user_context, instance_ref['id']) - def test_update_available_resource_works_correctly(self): + def tes1t_update_available_resource_works_correctly(self): """Confirm compute_node table is updated successfully.""" org_path = FLAGS.instances_path = '' FLAGS.instances_path = '.' + # Prepare mocks + def getVersion(): + return 12003 + + def getType(): + return 'qemu' + + def listDomainsID(): + return [] + service_ref = self.create_service(host='dummy') - self.create_fake_libvirt_mock() + self.create_fake_libvirt_mock(getVersion=getVersion, + getType=getType, + listDomainsID=listDomainsID) self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, 'get_cpu_info') libvirt_conn.LibvirtConnection.get_cpu_info().AndReturn('cpuinfo') + # Start test self.mox.ReplayAll() conn = libvirt_conn.LibvirtConnection(False) conn.update_available_resource(self.context, 'dummy') service_ref = db.service_get(self.context, service_ref['id']) compute_node = service_ref['compute_node'][0] - c1 = (compute_node['vcpus'] > 0) - c2 = (compute_node['memory_mb'] > 0) - c3 = (compute_node['local_gb'] > 0) - c4 = (compute_node['vcpus_used'] == 0) - c5 = (compute_node['memory_mb_used'] > 0) - c6 = (compute_node['local_gb_used'] > 0) - c7 = (len(compute_node['hypervisor_type']) > 0) - c8 = (compute_node['hypervisor_version'] > 0) - - self.assertTrue(c1 and c2 and c3 and c4 and c5 and c6 and c7 and c8) + if sys.platform.upper() == 'LINUX2': + self.assertTrue(compute_node['vcpus'] > 0) + self.assertTrue(compute_node['memory_mb'] > 0) + self.assertTrue(compute_node['local_gb'] > 0) + self.assertTrue(compute_node['vcpus_used'] == 0) + self.assertTrue(compute_node['memory_mb_used'] > 0) + self.assertTrue(compute_node['local_gb_used'] > 0) + self.assertTrue(len(compute_node['hypervisor_type']) > 0) + self.assertTrue(compute_node['hypervisor_version'] > 0) + else: + self.assertTrue(compute_node['vcpus'] > 0) + self.assertTrue(compute_node['memory_mb'] == 0) + self.assertTrue(compute_node['local_gb'] > 0) + self.assertTrue(compute_node['vcpus_used'] == 0) + self.assertTrue(compute_node['memory_mb_used'] == 0) + self.assertTrue(compute_node['local_gb_used'] > 0) + self.assertTrue(len(compute_node['hypervisor_type']) > 0) + self.assertTrue(compute_node['hypervisor_version'] > 0) db.service_destroy(self.context, service_ref['id']) FLAGS.instances_path = org_path @@ -319,6 +350,84 @@ class LibvirtConnTestCase(test.TestCase): FLAGS.instances_path = org_path + def test_ensure_filtering_rules_for_instance_timeout(self): + """ensure_filtering_fules_for_instance() finishes with timeout.""" + # Skip if non-libvirt environment + if not self.lazy_load_library_exists(): + return + + # Preparing mocks + def fake_none(self): + return + + def fake_raise(self): + raise libvirt.libvirtError('ERR') + + self.create_fake_libvirt_mock(nwfilterLookupByName=fake_raise) + instance_ref = db.instance_create(self.context, self.test_instance) + + # Start test + self.mox.ReplayAll() + try: + conn = libvirt_conn.LibvirtConnection(False) + conn.firewall_driver.setattr('setup_basic_filtering', fake_none) + conn.firewall_driver.setattr('prepare_instance_filter', fake_none) + conn.ensure_filtering_rules_for_instance(instance_ref) + except exception.Error, e: + c1 = (0 <= e.message.find('Timeout migrating for')) + self.assertTrue(c1) + + db.instance_destroy(self.context, instance_ref['id']) + + def test_live_migration_raises_exception(self): + """Confirms recover method is called when exceptions are raised.""" + # Skip if non-libvirt environment + if not self.lazy_load_library_exists(): + return + + # Preparing data + self.compute = utils.import_object(FLAGS.compute_manager) + instance_dict = {'host': 'fake', 'state': power_state.RUNNING, + 'state_description': 'running'} + instance_ref = db.instance_create(self.context, self.test_instance) + instance_ref = db.instance_update(self.context, instance_ref['id'], + instance_dict) + vol_dict = {'status': 'migrating', 'size': 1} + volume_ref = db.volume_create(self.context, vol_dict) + db.volume_attached(self.context, volume_ref['id'], instance_ref['id'], + '/dev/fake') + + # Preparing mocks + vdmock = self.mox.CreateMock(libvirt.virDomain) + self.mox.StubOutWithMock(vdmock, "migrateToURI") + vdmock.migrateToURI(FLAGS.live_migration_uri % 'dest', + mox.IgnoreArg(), + None, FLAGS.live_migration_bandwidth).\ + AndRaise(libvirt.libvirtError('ERR')) + + def fake_lookup(instance_name): + if instance_name == instance_ref.name: + return vdmock + + self.create_fake_libvirt_mock(lookupByName=fake_lookup) + + # Start test + self.mox.ReplayAll() + conn = libvirt_conn.LibvirtConnection(False) + self.assertRaises(libvirt.libvirtError, + conn._live_migration, + self.context, instance_ref, 'dest', '', + self.compute.recover_live_migration) + + instance_ref = db.instance_get(self.context, instance_ref['id']) + self.assertTrue(instance_ref['state_description'] == 'running') + self.assertTrue(instance_ref['state'] == power_state.RUNNING) + volume_ref = db.volume_get(self.context, volume_ref['id']) + self.assertTrue(volume_ref['status'] == 'in-use') + + db.volume_destroy(self.context, volume_ref['id']) + db.instance_destroy(self.context, instance_ref['id']) + def tearDown(self): self.manager.delete_project(self.project) self.manager.delete_user(self.user) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 71ca508b0..627a12a1c 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -36,8 +36,10 @@ Supports KVM, QEMU, UML, and XEN. """ +import multiprocessing import os import shutil +import sys import random import subprocess import time @@ -858,7 +860,7 @@ class LibvirtConnection(object): """ - return open('/proc/cpuinfo').read().count('processor') + return multiprocessing.cpu_count() def get_memory_mb_total(self): """Get the total memory size(MB) of physical computer. @@ -867,6 +869,9 @@ class LibvirtConnection(object): """ + if sys.platform.upper() != 'LINUX2': + return 0 + meminfo = open('/proc/meminfo').read().split() idx = meminfo.index('MemTotal:') # transforming kb to mb. @@ -905,6 +910,9 @@ class LibvirtConnection(object): """ + if sys.platform.upper() != 'LINUX2': + return 0 + m = open('/proc/meminfo').read().split() idx1 = m.index('MemFree:') idx2 = m.index('Buffers:') @@ -1126,7 +1134,7 @@ class LibvirtConnection(object): # wait for completion timeout_count = range(FLAGS.live_migration_retry_count) - while not timeout_count: + while timeout_count: try: filter_name = 'nova-instance-%s' % instance_ref.name self._conn.nwfilterLookupByName(filter_name) -- cgit From 23291a5e1a0134aff5fe030b52d4335a6f2a18d9 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Sat, 5 Mar 2011 01:07:12 +0900 Subject: delete unnecessary DECLARE --- nova/tests/test_virt.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 5bb31659b..7ea8c0fb5 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -36,7 +36,6 @@ from nova.virt import libvirt_conn libvirt = None FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') -flags.DECLARE('compute_driver', 'nova.compute.manager') class LibvirtConnTestCase(test.TestCase): -- cgit From f36b4fe22bcb187d5f426320bbe43fcf3cb1a30a Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Fri, 4 Mar 2011 14:44:29 -0500 Subject: refactor server tests to support xml and json separately --- nova/tests/api/openstack/test_servers.py | 103 ++++++++++++++++++++----------- 1 file changed, 68 insertions(+), 35 deletions(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index bf934113a..9e7bc3aac 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -233,12 +233,9 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) - def _personality_dict(self, path, contents): - return {'path': path, 'contents': contents} + def _setup_mock_compute_api_for_personality(self): - def _create_instance_with_personality(self, personality): - - class FakeComputeAPI(object): + class MockComputeAPI(object): def __init__(self): self.personality_files = None @@ -255,28 +252,74 @@ class ServersTest(test.TestCase): return canned_return return stub_method - compute_api = FakeComputeAPI() + compute_api = MockComputeAPI() self.stubs.Set(nova.compute, 'API', make_stub_method(compute_api)) self.stubs.Set(nova.api.openstack.servers.Controller, '_get_kernel_ramdisk_from_image', make_stub_method((1, 1))) self.stubs.Set(nova.api.openstack.common, 'get_image_id_from_image_hash', make_stub_method(2)) - body = dict(server=dict( - name='server_test', imageId=2, flavorId=2, - metadata={}, - personality=personality)) - if personality is None: - del body['server']['personality'] - + return compute_api + + def _create_personality_request_dict(self, personality_files): + server = {} + server['name'] = 'new-server-test' + server['imageId'] = 1 + server['flavorId'] = 1 + if personality_files is not None: + personalities = [] + for path, contents in personality_files: + personalities.append({'path': path, 'contents': contents}) + server['personality'] = personalities + return {'server': server} + + def _create_personality_request_json(self, personality_files): + body_dict = self._create_personality_request_dict(personality_files) req = webob.Request.blank('/v1.0/servers') + req.content_type = 'application/json' req.method = 'POST' - req.body = json.dumps(body) - return (req, req.get_response(fakes.wsgi_app()), - compute_api.personality_files) + req.body = json.dumps(body_dict) + return req + + def _format_xml_request_body(self, body_dict): + server = body_dict['server'] + body_parts = [] + body_parts.extend([ + '', + '' % ( + server['name'], server['imageId'], server['flavorId'])]) + if 'metadata' in server: + metadata = server['metadata'] + body_parts.append('') + for item in metadata.iteritems(): + body_parts.append('%s' % item) + body_parts.append('') + if 'personality' in server: + personalities = server['personality'] + body_parts.append('') + for item in personalities.iteritems(): + body_parts.append('%s' % item) + body_parts.append('') + body_parts.append('') + return ''.join(body_parts) + + def _create_personality_request_xml(self, personality_files): + body_dict = self._create_personality_request_dict(personality_files) + req = webob.Request.blank('/v1.0/servers') + req.content_type = 'application/xml' + req.method = 'POST' + req.body = self._format_xml_request_body(body_dict) + return req + + def _create_instance_with_personality_json(self, personality): + compute_api = self._setup_mock_compute_api_for_personality() + request = self._create_personality_request_json(personality) + response = request.get_response(fakes.wsgi_app()) + return (request, response, compute_api.personality_files) def test_create_instance_with_no_personality(self): request, response, personality_files = \ - self._create_instance_with_personality(personality=None) + self._create_instance_with_personality_json(personality=None) self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, []) @@ -284,18 +327,18 @@ class ServersTest(test.TestCase): path = '/my/file/path' contents = '#!/bin/bash\necho "Hello, World!"\n' b64contents = base64.b64encode(contents) - personality = [self._personality_dict(path, b64contents)] + personality = [(path, b64contents)] request, response, personality_files = \ - self._create_instance_with_personality(personality) + self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, [(path, contents)]) def test_create_instance_with_personality_with_non_b64_content(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Oh no!"\n' - personality = [self._personality_dict(path, contents)] + personality = [(path, contents)] request, response, personality_files = \ - self._create_instance_with_personality(personality) + self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 400) self.assertEquals(personality_files, None) @@ -307,31 +350,21 @@ class ServersTest(test.TestCase): ] personality = [] for path, content in files: - personality.append(self._personality_dict( - path, base64.b64encode(content))) + personality.append((path, base64.b64encode(content))) request, response, personality_files = \ - self._create_instance_with_personality(personality) + self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, files) def test_create_instance_personality_empty_content(self): path = '/my/file/path' contents = '' - personality = [self._personality_dict(path, contents)] + personality = [(path, contents)] request, response, personality_files = \ - self._create_instance_with_personality(personality) + self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, [(path, contents)]) - def test_create_instance_personality_not_a_list(self): - path = '/my/file/path' - contents = 'myfilecontents' - personality = self._personality_dict(path, contents) - request, response, personality_files = \ - self._create_instance_with_personality(personality) - self.assertEquals(response.status_int, 400) - self.assertEquals(personality_files, None) - def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' -- cgit From a38e6c67c37a4d3336cf1dc3717fd5612a474183 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Fri, 4 Mar 2011 14:45:31 -0500 Subject: remove xml testing infrastructure since it is not feasible to use at present --- nova/tests/api/openstack/test_servers.py | 31 ------------------------------- 1 file changed, 31 deletions(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 9e7bc3aac..c125e6192 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -280,37 +280,6 @@ class ServersTest(test.TestCase): req.body = json.dumps(body_dict) return req - def _format_xml_request_body(self, body_dict): - server = body_dict['server'] - body_parts = [] - body_parts.extend([ - '', - '' % ( - server['name'], server['imageId'], server['flavorId'])]) - if 'metadata' in server: - metadata = server['metadata'] - body_parts.append('') - for item in metadata.iteritems(): - body_parts.append('%s' % item) - body_parts.append('') - if 'personality' in server: - personalities = server['personality'] - body_parts.append('') - for item in personalities.iteritems(): - body_parts.append('%s' % item) - body_parts.append('') - body_parts.append('') - return ''.join(body_parts) - - def _create_personality_request_xml(self, personality_files): - body_dict = self._create_personality_request_dict(personality_files) - req = webob.Request.blank('/v1.0/servers') - req.content_type = 'application/xml' - req.method = 'POST' - req.body = self._format_xml_request_body(body_dict) - return req - def _create_instance_with_personality_json(self, personality): compute_api = self._setup_mock_compute_api_for_personality() request = self._create_personality_request_json(personality) -- cgit From 10668b87f46a1fb5d039f6e7d7a7a55b89d7602a Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Fri, 4 Mar 2011 17:04:41 -0500 Subject: respond well if personality attribute is incomplete --- nova/api/openstack/servers.py | 3 ++ nova/tests/api/openstack/test_servers.py | 48 ++++++++++++++++++++++++++++---- 2 files changed, 46 insertions(+), 5 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 8b7b20b92..7c620dbc6 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -153,6 +153,9 @@ class Controller(wsgi.Controller): try: path = item['path'] contents = item['contents'] + except KeyError, key: + expl = 'Bad personality format: missing %s' % key + raise exc.HTTPBadRequest(explanation=expl) except TypeError: raise exc.HTTPBadRequest(explanation='Bad personality format') try: diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index c125e6192..8fb5a9aec 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -272,19 +272,24 @@ class ServersTest(test.TestCase): server['personality'] = personalities return {'server': server} - def _create_personality_request_json(self, personality_files): - body_dict = self._create_personality_request_dict(personality_files) + def _get_create_request_json(self, body_dict): req = webob.Request.blank('/v1.0/servers') req.content_type = 'application/json' req.method = 'POST' req.body = json.dumps(body_dict) return req - def _create_instance_with_personality_json(self, personality): + def _run_create_instance_with_mock_compute_api(self, request): compute_api = self._setup_mock_compute_api_for_personality() - request = self._create_personality_request_json(personality) response = request.get_response(fakes.wsgi_app()) - return (request, response, compute_api.personality_files) + return compute_api, response + + def _create_instance_with_personality_json(self, personality): + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + return request, response, compute_api.personality_files def test_create_instance_with_no_personality(self): request, response, personality_files = \ @@ -302,6 +307,39 @@ class ServersTest(test.TestCase): self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, [(path, contents)]) + def test_create_instance_with_personality_no_path(self): + personality = [('/remove/this/path', + base64.b64encode('my\n\file\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + del body_dict['server']['personality'][0]['path'] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + + def test_create_instance_with_personality_no_contents(self): + personality = [('/test/path', + base64.b64encode('remove\nthese\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + del body_dict['server']['personality'][0]['contents'] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + + def test_create_instance_with_personality_not_a_list(self): + personality = [('/test/path', base64.b64encode('test\ncontents\n'))] + body_dict = self._create_personality_request_dict(personality) + body_dict['server']['personality'] = \ + body_dict['server']['personality'][0] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + def test_create_instance_with_personality_with_non_b64_content(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Oh no!"\n' -- cgit From 7af17cbef6f3e1c5b052133e40e0edbd8ca9ffb3 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Sun, 6 Mar 2011 10:41:24 -0500 Subject: select cleanups --- nova/api/openstack/servers.py | 2 +- nova/quota.py | 14 ++++++-------- nova/virt/xenapi/vmops.py | 2 +- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 7c620dbc6..93f504f91 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -153,7 +153,7 @@ class Controller(wsgi.Controller): try: path = item['path'] contents = item['contents'] - except KeyError, key: + except KeyError as key: expl = 'Bad personality format: missing %s' % key raise exc.HTTPBadRequest(explanation=expl) except TypeError: diff --git a/nova/quota.py b/nova/quota.py index 4b777624c..1d79f9ef2 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -46,14 +46,12 @@ flags.DEFINE_integer('quota_personality_max_path_bytes', 255, def get_quota(context, project_id): - rval = { - 'instances': FLAGS.quota_instances, - 'cores': FLAGS.quota_cores, - 'volumes': FLAGS.quota_volumes, - 'gigabytes': FLAGS.quota_gigabytes, - 'floating_ips': FLAGS.quota_floating_ips, - 'metadata_items': FLAGS.quota_metadata_items, - } + rval = {'instances': FLAGS.quota_instances, + 'cores': FLAGS.quota_cores, + 'volumes': FLAGS.quota_volumes, + 'gigabytes': FLAGS.quota_gigabytes, + 'floating_ips': FLAGS.quota_floating_ips, + 'metadata_items': FLAGS.quota_metadata_items} try: quota = db.quota_get(context, project_id) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 89d58a664..cf4bedaa9 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -316,7 +316,7 @@ class VMOps(object): def inject_file(self, instance, path, contents): """Write a file to the VM instance. The path to which it is to be - written and the contents of the file need to be supplied; both should + written and the contents of the file need to be supplied; both will be base64-encoded to prevent errors with non-ASCII characters being transmitted. If the agent does not support file injection, or the user has disabled it, a NotImplementedError will be raised. -- cgit From aa4b8a557505108341d603659a5456d10d5f9632 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Sun, 6 Mar 2011 12:06:39 -0500 Subject: avoid possible string/int comparison problems --- nova/quota.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/quota.py b/nova/quota.py index 1d79f9ef2..fdef42bc5 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -115,17 +115,17 @@ def allowed_metadata_items(context, num_metadata_items): def allowed_personality_files(context): """Return the number of personality files allowed""" - return FLAGS.quota_personality_max_files + return int(FLAGS.quota_personality_max_files) def allowed_personality_content_bytes(context): """Return the number of bytes allowed per personality content""" - return FLAGS.quota_personality_max_content_bytes + return int(FLAGS.quota_personality_max_content_bytes) def allowed_personality_path_bytes(context): """Return the number of bytes allowed in a personality file path""" - return FLAGS.quota_personality_max_path_bytes + return int(FLAGS.quota_personality_max_path_bytes) class QuotaError(exception.ApiError): -- cgit From f1dea606a64c9144fb723be0e5b86806891380f8 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 9 Mar 2011 11:54:21 -0500 Subject: add override to handle xml deserialization for server instance creation --- nova/api/openstack/servers.py | 68 +++++++- nova/tests/api/openstack/fakes.py | 2 - nova/tests/api/openstack/test_servers.py | 277 +++++++++++++++++++++++++++++++ 3 files changed, 344 insertions(+), 3 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 93f504f91..4a6282204 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -17,6 +17,7 @@ import base64 import hashlib import json import traceback +from xml.dom import minidom from webob import exc @@ -166,9 +167,16 @@ class Controller(wsgi.Controller): personality_files.append((path, contents)) return personality_files + def _deserialize_create(self, request): + if request.content_type == "application/xml": + deserializer = ServerCreateRequestXMLDeserializer() + return deserializer.deserialize(request.body) + else: + return self._deserialize(request.body, request) + def create(self, req): """ Creates a new server for a given user """ - env = self._deserialize(req.body, req) + env = self._deserialize_create(req) if not env: return faults.Fault(exc.HTTPUnprocessableEntity()) @@ -448,3 +456,61 @@ class Controller(wsgi.Controller): _("Ramdisk not found for image %(image_id)s") % locals()) return kernel_id, ramdisk_id + + +class ServerCreateRequestXMLDeserializer(object): + + def deserialize(self, string): + dom = minidom.parseString(string) + server = self._extract_server(dom) + return {'server': server} + + def _extract_server(self, node): + server = {} + server_node = self._find_first_child_named(node, 'server') + for attr in ["name", "imageId", "flavorId"]: + server[attr] = server_node.getAttribute(attr) + metadata = self._extract_metadata(server_node) + if metadata is not None: + server["metadata"] = metadata + personality = self._extract_personality(server_node) + if personality is not None: + server["personality"] = personality + return server + + def _extract_metadata(self, server_node): + metadata_node = self._find_first_child_named(server_node, "metadata") + if metadata_node is None: + return None + metadata = {} + for meta_node in metadata_node.childNodes: + key = meta_node.getAttribute("key") + metadata[key] = self._extract_text(meta_node) + return metadata + + def _extract_personality(self, server_node): + personality_node = \ + self._find_first_child_named(server_node, "personality") + if personality_node is None: + return None + personality = [] + for file_node in personality_node.childNodes: + item = {} + if file_node.hasAttribute("path"): + item["path"] = file_node.getAttribute("path") + item["contents"] = self._extract_text(file_node) + personality.append(item) + return personality + + def _find_first_child_named(self, parent, name): + for node in parent.childNodes: + if node.nodeName == name: + return node + return None + + def _extract_text(self, node): + if len(node.childNodes) == 1: + child = node.childNodes[0] + if child.nodeType == child.TEXT_NODE: + return child.nodeValue + return "" diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 49ce8c1b5..1d8692528 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -68,8 +68,6 @@ def fake_auth_init(self, application): @webob.dec.wsgify def fake_wsgi(self, req): req.environ['nova.context'] = context.RequestContext(1, 1) - if req.body: - req.environ['inst_dict'] = json.loads(req.body) return self.application diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 8fb5a9aec..5132d1ad3 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -18,6 +18,7 @@ import base64 import datetime import json +import unittest import stubout import webob @@ -284,6 +285,37 @@ class ServersTest(test.TestCase): response = request.get_response(fakes.wsgi_app()) return compute_api, response + def _format_xml_request_body(self, body_dict): + server = body_dict['server'] + body_parts = [] + body_parts.extend([ + '', + '' % ( + server['name'], server['imageId'], server['flavorId'])]) + if 'metadata' in server: + metadata = server['metadata'] + body_parts.append('') + for item in metadata.iteritems(): + body_parts.append('%s' % item) + body_parts.append('') + if 'personality' in server: + personalities = server['personality'] + body_parts.append('') + for file in personalities: + item = (file['path'], file['contents']) + body_parts.append('%s' % item) + body_parts.append('') + body_parts.append('') + return ''.join(body_parts) + + def _get_create_request_xml(self, body_dict): + req = webob.Request.blank('/v1.0/servers') + req.content_type = 'application/xml' + req.method = 'POST' + req.body = self._format_xml_request_body(body_dict) + return req + def _create_instance_with_personality_json(self, personality): body_dict = self._create_personality_request_dict(personality) request = self._get_create_request_json(body_dict) @@ -291,12 +323,25 @@ class ServersTest(test.TestCase): self._run_create_instance_with_mock_compute_api(request) return request, response, compute_api.personality_files + def _create_instance_with_personality_xml(self, personality): + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_xml(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + return request, response, compute_api.personality_files + def test_create_instance_with_no_personality(self): request, response, personality_files = \ self._create_instance_with_personality_json(personality=None) self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, []) + def test_create_instance_with_no_personality_xml(self): + request, response, personality_files = \ + self._create_instance_with_personality_xml(personality=None) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, []) + def test_create_instance_with_personality(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Hello, World!"\n' @@ -307,6 +352,16 @@ class ServersTest(test.TestCase): self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, [(path, contents)]) + def test_create_instance_with_personality_xml(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Hello, World!"\n' + b64contents = base64.b64encode(contents) + personality = [(path, b64contents)] + request, response, personality_files = \ + self._create_instance_with_personality_xml(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, [(path, contents)]) + def test_create_instance_with_personality_no_path(self): personality = [('/remove/this/path', base64.b64encode('my\n\file\ncontents'))] @@ -318,6 +373,17 @@ class ServersTest(test.TestCase): self.assertEquals(response.status_int, 400) self.assertEquals(compute_api.personality_files, None) + def _test_create_instance_with_personality_no_path_xml(self): + personality = [('/remove/this/path', + base64.b64encode('my\n\file\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_xml(body_dict) + request.body = request.body.replace(' path="/remove/this/path"', '') + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + def test_create_instance_with_personality_no_contents(self): personality = [('/test/path', base64.b64encode('remove\nthese\ncontents'))] @@ -605,3 +671,214 @@ class ServersTest(test.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '202 Accepted') self.assertEqual(self.server_delete_called, True) + + +class TestServerCreateRequestXMLDeserializer(unittest.TestCase): + + def setUp(self): + self.deserializer = servers.ServerCreateRequestXMLDeserializer() + + def test_minimal_request(self): + serial_request = """ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + }} + self.assertEquals(request, expected) + + def test_request_with_empty_metadata(self): + serial_request = """ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "metadata": {}, + }} + self.assertEquals(request, expected) + + def test_request_with_empty_personality(self): + serial_request = """ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "personality": [], + }} + self.assertEquals(request, expected) + + def test_request_with_empty_metadata_and_personality(self): + serial_request = """ +\ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "metadata": {}, + "personality": [], + }} + self.assertEquals(request, expected) + + def test_request_with_empty_metadata_and_personality_reversed(self): + serial_request = """ +\ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "metadata": {}, + "personality": [], + }} + self.assertEquals(request, expected) + + def test_request_with_one_personality(self): + serial_request = """ +\ +aabbccdd""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": "aabbccdd"}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_two_personalities(self): + serial_request = """ +\ +aabbccdd\ +abcd""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": "aabbccdd"}, + {"path": "/etc/sudoers", "contents": "abcd"}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_one_personality_missing_path(self): + serial_request = """ +\ +aabbccdd""" + request = self.deserializer.deserialize(serial_request) + expected = [{"contents": "aabbccdd"}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_one_personality_empty_contents(self): + serial_request = """ +\ +""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": ""}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_one_personality_empty_contents_variation(self): + serial_request = """ +\ +""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": ""}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_one_metadata(self): + serial_request = """ +\ +beta""" + request = self.deserializer.deserialize(serial_request) + expected = {"alpha": "beta"} + self.assertEquals(request["server"]["metadata"], expected) + + def test_request_with_two_metadata(self): + serial_request = """ +\ +betabar\ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"alpha": "beta", "foo": "bar"} + self.assertEquals(request["server"]["metadata"], expected) + + def test_request_with_metadata_missing_value(self): + serial_request = """ +\ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"alpha": ""} + self.assertEquals(request["server"]["metadata"], expected) + + def test_request_with_metadata_missing_key(self): + serial_request = """ +\ +beta""" + request = self.deserializer.deserialize(serial_request) + expected = {"": "beta"} + self.assertEquals(request["server"]["metadata"], expected) + + def test_request_with_metadata_duplicate_key(self): + serial_request = """ +\ +barbaz\ +""" + request = self.deserializer.deserialize(serial_request) + expected = {"foo": "baz"} + self.assertEquals(request["server"]["metadata"], expected) + + def test_canonical_request_from_docs(self): + serial_request = """ +\ +Apache1\ +\ +ICAgICAgDQoiQSBjbG91ZCBkb2VzIG5vdCBrbm93IHdoeSBp\ +dCBtb3ZlcyBpbiBqdXN0IHN1Y2ggYSBkaXJlY3Rpb24gYW5k\ +IGF0IHN1Y2ggYSBzcGVlZC4uLkl0IGZlZWxzIGFuIGltcHVs\ +c2lvbi4uLnRoaXMgaXMgdGhlIHBsYWNlIHRvIGdvIG5vdy4g\ +QnV0IHRoZSBza3kga25vd3MgdGhlIHJlYXNvbnMgYW5kIHRo\ +ZSBwYXR0ZXJucyBiZWhpbmQgYWxsIGNsb3VkcywgYW5kIHlv\ +dSB3aWxsIGtub3csIHRvbywgd2hlbiB5b3UgbGlmdCB5b3Vy\ +c2VsZiBoaWdoIGVub3VnaCB0byBzZWUgYmV5b25kIGhvcml6\ +b25zLiINCg0KLVJpY2hhcmQgQmFjaA==\ +""" + expected = {"server": { + "name": "new-server-test", + "imageId": "1", + "flavorId": "1", + "metadata": { + "My Server Name": "Apache1", + }, + "personality": [ + { + "path": "/etc/banner.txt", + "contents": """\ +ICAgICAgDQoiQSBjbG91ZCBkb2VzIG5vdCBrbm93IHdoeSBp\ +dCBtb3ZlcyBpbiBqdXN0IHN1Y2ggYSBkaXJlY3Rpb24gYW5k\ +IGF0IHN1Y2ggYSBzcGVlZC4uLkl0IGZlZWxzIGFuIGltcHVs\ +c2lvbi4uLnRoaXMgaXMgdGhlIHBsYWNlIHRvIGdvIG5vdy4g\ +QnV0IHRoZSBza3kga25vd3MgdGhlIHJlYXNvbnMgYW5kIHRo\ +ZSBwYXR0ZXJucyBiZWhpbmQgYWxsIGNsb3VkcywgYW5kIHlv\ +dSB3aWxsIGtub3csIHRvbywgd2hlbiB5b3UgbGlmdCB5b3Vy\ +c2VsZiBoaWdoIGVub3VnaCB0byBzZWUgYmV5b25kIGhvcml6\ +b25zLiINCg0KLVJpY2hhcmQgQmFjaA==""", + } + ], + }} + request = self.deserializer.deserialize(serial_request) + self.assertEqual(request, expected) + -- cgit From 1d74816bf705cb672d9d323398b03142297d8bec Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 9 Mar 2011 12:03:17 -0500 Subject: remove superfluous trailing blank line --- nova/tests/api/openstack/test_servers.py | 1 - 1 file changed, 1 deletion(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 5132d1ad3..51fb09b18 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -881,4 +881,3 @@ b25zLiINCg0KLVJpY2hhcmQgQmFjaA==""", }} request = self.deserializer.deserialize(serial_request) self.assertEqual(request, expected) - -- cgit From 1166e16d08769222189e31e6de1c6019495fc743 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 9 Mar 2011 13:54:30 -0500 Subject: move my tests into their own testcase --- nova/tests/api/openstack/test_servers.py | 447 +++++++++++++++++-------------- 1 file changed, 243 insertions(+), 204 deletions(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index f2fbb04af..b0f888766 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -238,210 +238,6 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) - def _setup_mock_compute_api_for_personality(self): - - class MockComputeAPI(object): - - def __init__(self): - self.personality_files = None - - def create(self, *args, **kwargs): - if 'personality_files' in kwargs: - self.personality_files = kwargs['personality_files'] - else: - self.personality_files = None - return [{'id': '1234', 'display_name': 'fakeinstance'}] - - def make_stub_method(canned_return): - def stub_method(*args, **kwargs): - return canned_return - return stub_method - - compute_api = MockComputeAPI() - self.stubs.Set(nova.compute, 'API', make_stub_method(compute_api)) - self.stubs.Set(nova.api.openstack.servers.Controller, - '_get_kernel_ramdisk_from_image', make_stub_method((1, 1))) - self.stubs.Set(nova.api.openstack.common, - 'get_image_id_from_image_hash', make_stub_method(2)) - return compute_api - - def _create_personality_request_dict(self, personality_files): - server = {} - server['name'] = 'new-server-test' - server['imageId'] = 1 - server['flavorId'] = 1 - if personality_files is not None: - personalities = [] - for path, contents in personality_files: - personalities.append({'path': path, 'contents': contents}) - server['personality'] = personalities - return {'server': server} - - def _get_create_request_json(self, body_dict): - req = webob.Request.blank('/v1.0/servers') - req.content_type = 'application/json' - req.method = 'POST' - req.body = json.dumps(body_dict) - return req - - def _run_create_instance_with_mock_compute_api(self, request): - compute_api = self._setup_mock_compute_api_for_personality() - response = request.get_response(fakes.wsgi_app()) - return compute_api, response - - def _format_xml_request_body(self, body_dict): - server = body_dict['server'] - body_parts = [] - body_parts.extend([ - '', - '' % ( - server['name'], server['imageId'], server['flavorId'])]) - if 'metadata' in server: - metadata = server['metadata'] - body_parts.append('') - for item in metadata.iteritems(): - body_parts.append('%s' % item) - body_parts.append('') - if 'personality' in server: - personalities = server['personality'] - body_parts.append('') - for file in personalities: - item = (file['path'], file['contents']) - body_parts.append('%s' % item) - body_parts.append('') - body_parts.append('') - return ''.join(body_parts) - - def _get_create_request_xml(self, body_dict): - req = webob.Request.blank('/v1.0/servers') - req.content_type = 'application/xml' - req.method = 'POST' - req.body = self._format_xml_request_body(body_dict) - return req - - def _create_instance_with_personality_json(self, personality): - body_dict = self._create_personality_request_dict(personality) - request = self._get_create_request_json(body_dict) - compute_api, response = \ - self._run_create_instance_with_mock_compute_api(request) - return request, response, compute_api.personality_files - - def _create_instance_with_personality_xml(self, personality): - body_dict = self._create_personality_request_dict(personality) - request = self._get_create_request_xml(body_dict) - compute_api, response = \ - self._run_create_instance_with_mock_compute_api(request) - return request, response, compute_api.personality_files - - def test_create_instance_with_no_personality(self): - request, response, personality_files = \ - self._create_instance_with_personality_json(personality=None) - self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, []) - - def test_create_instance_with_no_personality_xml(self): - request, response, personality_files = \ - self._create_instance_with_personality_xml(personality=None) - self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, []) - - def test_create_instance_with_personality(self): - path = '/my/file/path' - contents = '#!/bin/bash\necho "Hello, World!"\n' - b64contents = base64.b64encode(contents) - personality = [(path, b64contents)] - request, response, personality_files = \ - self._create_instance_with_personality_json(personality) - self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, [(path, contents)]) - - def test_create_instance_with_personality_xml(self): - path = '/my/file/path' - contents = '#!/bin/bash\necho "Hello, World!"\n' - b64contents = base64.b64encode(contents) - personality = [(path, b64contents)] - request, response, personality_files = \ - self._create_instance_with_personality_xml(personality) - self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, [(path, contents)]) - - def test_create_instance_with_personality_no_path(self): - personality = [('/remove/this/path', - base64.b64encode('my\n\file\ncontents'))] - body_dict = self._create_personality_request_dict(personality) - del body_dict['server']['personality'][0]['path'] - request = self._get_create_request_json(body_dict) - compute_api, response = \ - self._run_create_instance_with_mock_compute_api(request) - self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) - - def _test_create_instance_with_personality_no_path_xml(self): - personality = [('/remove/this/path', - base64.b64encode('my\n\file\ncontents'))] - body_dict = self._create_personality_request_dict(personality) - request = self._get_create_request_xml(body_dict) - request.body = request.body.replace(' path="/remove/this/path"', '') - compute_api, response = \ - self._run_create_instance_with_mock_compute_api(request) - self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) - - def test_create_instance_with_personality_no_contents(self): - personality = [('/test/path', - base64.b64encode('remove\nthese\ncontents'))] - body_dict = self._create_personality_request_dict(personality) - del body_dict['server']['personality'][0]['contents'] - request = self._get_create_request_json(body_dict) - compute_api, response = \ - self._run_create_instance_with_mock_compute_api(request) - self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) - - def test_create_instance_with_personality_not_a_list(self): - personality = [('/test/path', base64.b64encode('test\ncontents\n'))] - body_dict = self._create_personality_request_dict(personality) - body_dict['server']['personality'] = \ - body_dict['server']['personality'][0] - request = self._get_create_request_json(body_dict) - compute_api, response = \ - self._run_create_instance_with_mock_compute_api(request) - self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) - - def test_create_instance_with_personality_with_non_b64_content(self): - path = '/my/file/path' - contents = '#!/bin/bash\necho "Oh no!"\n' - personality = [(path, contents)] - request, response, personality_files = \ - self._create_instance_with_personality_json(personality) - self.assertEquals(response.status_int, 400) - self.assertEquals(personality_files, None) - - def test_create_instance_with_three_personalities(self): - files = [ - ('/etc/sudoers', 'ALL ALL=NOPASSWD: ALL\n'), - ('/etc/motd', 'Enjoy your root access!\n'), - ('/etc/dovecot.conf', 'dovecot\nconfig\nstuff\n'), - ] - personality = [] - for path, content in files: - personality.append((path, base64.b64encode(content))) - request, response, personality_files = \ - self._create_instance_with_personality_json(personality) - self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, files) - - def test_create_instance_personality_empty_content(self): - path = '/my/file/path' - contents = '' - personality = [(path, contents)] - request, response, personality_files = \ - self._create_instance_with_personality_json(personality) - self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, [(path, contents)]) - def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' @@ -980,5 +776,248 @@ b25zLiINCg0KLVJpY2hhcmQgQmFjaA==""", self.assertEqual(request, expected) +class TestServerInstanceCreation(test.TestCase): + + def setUp(self): + super(TestServerInstanceCreation, self).setUp() + self.stubs = stubout.StubOutForTesting() + fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthDatabase.data = {} + fakes.stub_out_networking(self.stubs) + fakes.stub_out_rate_limiting(self.stubs) + fakes.stub_out_auth(self.stubs) + fakes.stub_out_key_pair_funcs(self.stubs) + fakes.stub_out_image_service(self.stubs) + self.stubs.Set(nova.db.api, 'instance_get_all', return_servers) + self.stubs.Set(nova.db.api, 'instance_get', return_server) + self.stubs.Set(nova.db.api, 'instance_get_all_by_user', + return_servers) + self.stubs.Set(nova.db.api, 'instance_add_security_group', + return_security_group) + self.stubs.Set(nova.db.api, 'instance_update', instance_update) + self.stubs.Set(nova.db.api, 'instance_get_fixed_address', + instance_address) + self.stubs.Set(nova.db.api, 'instance_get_floating_address', + instance_address) + self.stubs.Set(nova.compute.API, 'pause', fake_compute_api) + self.stubs.Set(nova.compute.API, 'unpause', fake_compute_api) + self.stubs.Set(nova.compute.API, 'suspend', fake_compute_api) + self.stubs.Set(nova.compute.API, 'resume', fake_compute_api) + self.stubs.Set(nova.compute.API, "get_diagnostics", fake_compute_api) + self.stubs.Set(nova.compute.API, "get_actions", fake_compute_api) + self.allow_admin = FLAGS.allow_admin_api + + self.webreq = common.webob_factory('/v1.0/servers') + + def tearDown(self): + self.stubs.UnsetAll() + FLAGS.allow_admin_api = self.allow_admin + super(TestServerInstanceCreation, self).tearDown() + + def _setup_mock_compute_api_for_personality(self): + + class MockComputeAPI(object): + + def __init__(self): + self.personality_files = None + + def create(self, *args, **kwargs): + if 'personality_files' in kwargs: + self.personality_files = kwargs['personality_files'] + else: + self.personality_files = None + return [{'id': '1234', 'display_name': 'fakeinstance'}] + + def make_stub_method(canned_return): + def stub_method(*args, **kwargs): + return canned_return + return stub_method + + compute_api = MockComputeAPI() + self.stubs.Set(nova.compute, 'API', make_stub_method(compute_api)) + self.stubs.Set(nova.api.openstack.servers.Controller, + '_get_kernel_ramdisk_from_image', make_stub_method((1, 1))) + self.stubs.Set(nova.api.openstack.common, + 'get_image_id_from_image_hash', make_stub_method(2)) + return compute_api + + def _create_personality_request_dict(self, personality_files): + server = {} + server['name'] = 'new-server-test' + server['imageId'] = 1 + server['flavorId'] = 1 + if personality_files is not None: + personalities = [] + for path, contents in personality_files: + personalities.append({'path': path, 'contents': contents}) + server['personality'] = personalities + return {'server': server} + + def _get_create_request_json(self, body_dict): + req = webob.Request.blank('/v1.0/servers') + req.content_type = 'application/json' + req.method = 'POST' + req.body = json.dumps(body_dict) + return req + + def _run_create_instance_with_mock_compute_api(self, request): + compute_api = self._setup_mock_compute_api_for_personality() + response = request.get_response(fakes.wsgi_app()) + return compute_api, response + + def _format_xml_request_body(self, body_dict): + server = body_dict['server'] + body_parts = [] + body_parts.extend([ + '', + '' % ( + server['name'], server['imageId'], server['flavorId'])]) + if 'metadata' in server: + metadata = server['metadata'] + body_parts.append('') + for item in metadata.iteritems(): + body_parts.append('%s' % item) + body_parts.append('') + if 'personality' in server: + personalities = server['personality'] + body_parts.append('') + for file in personalities: + item = (file['path'], file['contents']) + body_parts.append('%s' % item) + body_parts.append('') + body_parts.append('') + return ''.join(body_parts) + + def _get_create_request_xml(self, body_dict): + req = webob.Request.blank('/v1.0/servers') + req.content_type = 'application/xml' + req.method = 'POST' + req.body = self._format_xml_request_body(body_dict) + return req + + def _create_instance_with_personality_json(self, personality): + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + return request, response, compute_api.personality_files + + def _create_instance_with_personality_xml(self, personality): + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_xml(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + return request, response, compute_api.personality_files + + def test_create_instance_with_no_personality(self): + request, response, personality_files = \ + self._create_instance_with_personality_json(personality=None) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, []) + + def test_create_instance_with_no_personality_xml(self): + request, response, personality_files = \ + self._create_instance_with_personality_xml(personality=None) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, []) + + def test_create_instance_with_personality(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Hello, World!"\n' + b64contents = base64.b64encode(contents) + personality = [(path, b64contents)] + request, response, personality_files = \ + self._create_instance_with_personality_json(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, [(path, contents)]) + + def test_create_instance_with_personality_xml(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Hello, World!"\n' + b64contents = base64.b64encode(contents) + personality = [(path, b64contents)] + request, response, personality_files = \ + self._create_instance_with_personality_xml(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, [(path, contents)]) + + def test_create_instance_with_personality_no_path(self): + personality = [('/remove/this/path', + base64.b64encode('my\n\file\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + del body_dict['server']['personality'][0]['path'] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + + def _test_create_instance_with_personality_no_path_xml(self): + personality = [('/remove/this/path', + base64.b64encode('my\n\file\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + request = self._get_create_request_xml(body_dict) + request.body = request.body.replace(' path="/remove/this/path"', '') + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + + def test_create_instance_with_personality_no_contents(self): + personality = [('/test/path', + base64.b64encode('remove\nthese\ncontents'))] + body_dict = self._create_personality_request_dict(personality) + del body_dict['server']['personality'][0]['contents'] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + + def test_create_instance_with_personality_not_a_list(self): + personality = [('/test/path', base64.b64encode('test\ncontents\n'))] + body_dict = self._create_personality_request_dict(personality) + body_dict['server']['personality'] = \ + body_dict['server']['personality'][0] + request = self._get_create_request_json(body_dict) + compute_api, response = \ + self._run_create_instance_with_mock_compute_api(request) + self.assertEquals(response.status_int, 400) + self.assertEquals(compute_api.personality_files, None) + + def test_create_instance_with_personality_with_non_b64_content(self): + path = '/my/file/path' + contents = '#!/bin/bash\necho "Oh no!"\n' + personality = [(path, contents)] + request, response, personality_files = \ + self._create_instance_with_personality_json(personality) + self.assertEquals(response.status_int, 400) + self.assertEquals(personality_files, None) + + def test_create_instance_with_three_personalities(self): + files = [ + ('/etc/sudoers', 'ALL ALL=NOPASSWD: ALL\n'), + ('/etc/motd', 'Enjoy your root access!\n'), + ('/etc/dovecot.conf', 'dovecot\nconfig\nstuff\n'), + ] + personality = [] + for path, content in files: + personality.append((path, base64.b64encode(content))) + request, response, personality_files = \ + self._create_instance_with_personality_json(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, files) + + def test_create_instance_personality_empty_content(self): + path = '/my/file/path' + contents = '' + personality = [(path, contents)] + request, response, personality_files = \ + self._create_instance_with_personality_json(personality) + self.assertEquals(response.status_int, 200) + self.assertEquals(personality_files, [(path, contents)]) + + if __name__ == "__main__": unittest.main() -- cgit From 3999bb363501c6587f75255333094c9e61bf1828 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 9 Mar 2011 14:07:33 -0500 Subject: remove unneeded stubs --- nova/tests/api/openstack/test_servers.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index b0f888766..c67ecdaae 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -783,32 +783,10 @@ class TestServerInstanceCreation(test.TestCase): self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} - fakes.stub_out_networking(self.stubs) - fakes.stub_out_rate_limiting(self.stubs) fakes.stub_out_auth(self.stubs) fakes.stub_out_key_pair_funcs(self.stubs) - fakes.stub_out_image_service(self.stubs) - self.stubs.Set(nova.db.api, 'instance_get_all', return_servers) - self.stubs.Set(nova.db.api, 'instance_get', return_server) - self.stubs.Set(nova.db.api, 'instance_get_all_by_user', - return_servers) - self.stubs.Set(nova.db.api, 'instance_add_security_group', - return_security_group) - self.stubs.Set(nova.db.api, 'instance_update', instance_update) - self.stubs.Set(nova.db.api, 'instance_get_fixed_address', - instance_address) - self.stubs.Set(nova.db.api, 'instance_get_floating_address', - instance_address) - self.stubs.Set(nova.compute.API, 'pause', fake_compute_api) - self.stubs.Set(nova.compute.API, 'unpause', fake_compute_api) - self.stubs.Set(nova.compute.API, 'suspend', fake_compute_api) - self.stubs.Set(nova.compute.API, 'resume', fake_compute_api) - self.stubs.Set(nova.compute.API, "get_diagnostics", fake_compute_api) - self.stubs.Set(nova.compute.API, "get_actions", fake_compute_api) self.allow_admin = FLAGS.allow_admin_api - self.webreq = common.webob_factory('/v1.0/servers') - def tearDown(self): self.stubs.UnsetAll() FLAGS.allow_admin_api = self.allow_admin -- cgit From 7854cf996608c09e753c70c3914746dab22c4e1a Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 9 Mar 2011 14:21:18 -0500 Subject: update authors file --- Authors | 1 + 1 file changed, 1 insertion(+) diff --git a/Authors b/Authors index 7993955e2..583cf8e75 100644 --- a/Authors +++ b/Authors @@ -39,6 +39,7 @@ Ken Pepple Kevin L. Mitchell Koji Iida Lorin Hochstein +Mark Washenberger Masanori Itoh Matt Dietz Michael Gundlach -- cgit From 2c733d5365b753989b506d82d376d980cd701547 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Wed, 9 Mar 2011 14:38:34 -0500 Subject: rearrange functions and add docstrings --- nova/api/openstack/servers.py | 83 ++++++++++++++++++++++++++----------------- 1 file changed, 50 insertions(+), 33 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 4a18d870c..419a001fb 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -141,39 +141,6 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def _get_personality_files(self, personality): - """ - Create a list of personality files from the personality attribute - - At this time, personality_files must be formatted as a list of - (file_path, file_content) pairs for compatibility with the - underlying compute service. - """ - personality_files = [] - for item in personality: - try: - path = item['path'] - contents = item['contents'] - except KeyError as key: - expl = 'Bad personality format: missing %s' % key - raise exc.HTTPBadRequest(explanation=expl) - except TypeError: - raise exc.HTTPBadRequest(explanation='Bad personality format') - try: - contents = base64.b64decode(contents) - except TypeError: - msg = 'Personality content for %s cannot be decoded' % path - raise exc.HTTPBadRequest(explanation=msg) - personality_files.append((path, contents)) - return personality_files - - def _deserialize_create(self, request): - if request.content_type == "application/xml": - deserializer = ServerCreateRequestXMLDeserializer() - return deserializer.deserialize(request.body) - else: - return self._deserialize(request.body, request) - def create(self, req): """ Creates a new server for a given user """ env = self._deserialize_create(req) @@ -218,6 +185,44 @@ class Controller(wsgi.Controller): personality_files=personality_files) return _translate_keys(instances[0]) + def _deserialize_create(self, request): + """ + Deserialize a create request + + Overrides normal behavior in the case of xml content + """ + if request.content_type == "application/xml": + deserializer = ServerCreateRequestXMLDeserializer() + return deserializer.deserialize(request.body) + else: + return self._deserialize(request.body, request) + + def _get_personality_files(self, personality): + """ + Create a list of personality files from the personality attribute + + At this time, personality_files must be formatted as a list of + (file_path, file_content) pairs for compatibility with the + underlying compute service. + """ + personality_files = [] + for item in personality: + try: + path = item['path'] + contents = item['contents'] + except KeyError as key: + expl = 'Bad personality format: missing %s' % key + raise exc.HTTPBadRequest(explanation=expl) + except TypeError: + raise exc.HTTPBadRequest(explanation='Bad personality format') + try: + contents = base64.b64decode(contents) + except TypeError: + msg = 'Personality content for %s cannot be decoded' % path + raise exc.HTTPBadRequest(explanation=msg) + personality_files.append((path, contents)) + return personality_files + def update(self, req, id): """ Updates the server name or password """ inst_dict = self._deserialize(req.body, req) @@ -507,13 +512,21 @@ class Controller(wsgi.Controller): class ServerCreateRequestXMLDeserializer(object): + """ + Deserializer to handle xml-formatted server create requests. + + Handles standard server attributes as well as optional metadata + and personality attributes + """ def deserialize(self, string): + """Deserialize an xml-formatted server create request""" dom = minidom.parseString(string) server = self._extract_server(dom) return {'server': server} def _extract_server(self, node): + """Marshal the server attribute of a parsed request""" server = {} server_node = self._find_first_child_named(node, 'server') for attr in ["name", "imageId", "flavorId"]: @@ -527,6 +540,7 @@ class ServerCreateRequestXMLDeserializer(object): return server def _extract_metadata(self, server_node): + """Marshal the metadata attribute of a parsed request""" metadata_node = self._find_first_child_named(server_node, "metadata") if metadata_node is None: return None @@ -537,6 +551,7 @@ class ServerCreateRequestXMLDeserializer(object): return metadata def _extract_personality(self, server_node): + """Marshal the personality attribute of a parsed request""" personality_node = \ self._find_first_child_named(server_node, "personality") if personality_node is None: @@ -551,12 +566,14 @@ class ServerCreateRequestXMLDeserializer(object): return personality def _find_first_child_named(self, parent, name): + """Search a nodes children for the first child with a given name""" for node in parent.childNodes: if node.nodeName == name: return node return None def _extract_text(self, node): + """Get the text field contained by the given node""" if len(node.childNodes) == 1: child = node.childNodes[0] if child.nodeType == child.TEXT_NODE: -- cgit From f0bb6d9fc47b92d335c7d7fa238dfd43f0dbdf69 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Thu, 10 Mar 2011 13:30:52 +0900 Subject: fixed based on reviewer's comment. --- bin/nova-manage | 8 ++--- nova/compute/manager.py | 22 ++++++------- nova/db/sqlalchemy/api.py | 4 +-- nova/scheduler/driver.py | 10 +++--- nova/tests/test_compute.py | 24 +++++++------- nova/tests/test_scheduler.py | 5 +-- nova/tests/test_service.py | 77 +++++++++++++++++--------------------------- nova/tests/test_virt.py | 6 ++-- nova/tests/test_volume.py | 7 ++-- nova/virt/libvirt_conn.py | 22 ++++++++----- nova/volume/driver.py | 2 +- 11 files changed, 89 insertions(+), 98 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index d782f6028..f9e4fa8dc 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -567,7 +567,7 @@ class VmCommands(object): if (FLAGS.volume_driver != 'nova.volume.driver.AOEDriver' and \ FLAGS.volume_driver != 'nova.volume.driver.ISCSIDriver'): msg = _("Support only AOEDriver and ISCSIDriver. Sorry!") - raise exception.Error(msg) + raise exception.Error(msg) rpc.call(ctxt, FLAGS.scheduler_topic, @@ -637,8 +637,8 @@ class ServiceCommands(object): "args": {"host": host}}) if type(result) != dict: - print 'Unexpected error occurs' - print '[Result]', result + print _('An unexpected error has occurred.') + print _('[Result]'), result else: cpu = result['resource']['vcpus'] mem = result['resource']['memory_mb'] @@ -667,7 +667,7 @@ class ServiceCommands(object): ctxt = context.get_admin_context() service_refs = db.service_get_all_by_host(ctxt, host) if len(service_refs) <= 0: - raise exception.Invalid(_('%s does not exists.') % host) + raise exception.Invalid(_('%s does not exist.') % host) service_refs = [s for s in service_refs if s['topic'] == 'compute'] if len(service_refs) <= 0: diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 3675cc92e..0cab10fc3 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -64,7 +64,7 @@ flags.DEFINE_integer('password_length', 12, flags.DEFINE_string('console_host', socket.gethostname(), 'Console proxy host to use to connect to instances on' 'this host.') -flags.DEFINE_string('live_migration_retry_count', 30, +flags.DEFINE_integer('live_migration_retry_count', 30, ("Retry count needed in live_migration." " sleep 1 sec for each count")) @@ -757,8 +757,9 @@ class ComputeManager(manager.Manager): dirpath = FLAGS.instances_path fd, tmp_file = tempfile.mkstemp(dir=dirpath) LOG.debug(_("Creating tmpfile %s to notify to other " - "compute node that they mounts same storage.") % tmp_file) - os.fdopen(fd, 'w+').close() + "compute nodes that they should mount " + "the same storage.") % tmp_file) + os.close(fd) return os.path.basename(tmp_file) @exception.wrap_exception @@ -812,7 +813,7 @@ class ComputeManager(manager.Manager): # Getting fixed ips fixed_ip = self.db.instance_get_fixed_address(context, instance_id) if not fixed_ip: - msg = _("%(instance_id)s(%(ec2_id)s) does'nt have fixed_ip") + msg = _("%(instance_id)s(%(ec2_id)s) does not have fixed_ip.") raise exception.NotFound(msg % locals()) # If any volume is mounted, prepare here. @@ -929,7 +930,7 @@ class ComputeManager(manager.Manager): floating_ip = self.db.instance_get_floating_address(ctxt, instance_id) if not floating_ip: - LOG.info(_('floating_ip is not found for %s'), i_name) + LOG.info(_('No floating_ip is found for %s.'), i_name) else: floating_ip_ref = self.db.floating_ip_get_by_address(ctxt, floating_ip) @@ -937,7 +938,7 @@ class ComputeManager(manager.Manager): floating_ip_ref['address'], {'host': dest}) except exception.NotFound: - LOG.info(_('Floating_ip is not found for %s'), i_name) + LOG.info(_('No floating_ip is found for %s.'), i_name) except: LOG.error(_("Live migration: Unexpected error:" "%s cannot inherit floating ip..") % i_name) @@ -945,12 +946,11 @@ class ComputeManager(manager.Manager): # Restore instance/volume state self.recover_live_migration(ctxt, instance_ref, dest) - LOG.info(_('Migrating %(i_name)s to %(dest)s finishes successfully.') + LOG.info(_('Migrating %(i_name)s to %(dest)s finished successfully.') % locals()) - LOG.info(_("The below error is normally occurs. " - "Just check if instance is successfully migrated.\n" - "libvir: QEMU error : Domain not found: no domain " - "with matching name..")) + LOG.info(_("You may see the error \"libvirt: QEMU error: " + "Domain not found: no domain with matching name.\" " + "This error can be safely ignored.")) def recover_live_migration(self, ctxt, instance_ref, host=None): """Recovers Instance/volume state from migrating -> running. diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 8ea5062ae..f44ca0fa3 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -192,8 +192,8 @@ def service_get_all_compute_by_host(context, host): all() if not result: - raise exception.NotFound(_("%s does not exist or not " - "compute node.") % host) + raise exception.NotFound(_("%s does not exist or is not " + "a compute node.") % host) return result diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 791f9000d..ed3dfe1c0 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -226,7 +226,6 @@ class Scheduler(object): "args": {'cpu_info': oservice_ref['cpu_info']}}) except rpc.RemoteError: - ec2_id = instance_ref['hostname'] src = instance_ref['host'] logging.exception(_("host %(dest)s is not compatible with " "original host %(src)s.") % locals()) @@ -259,9 +258,10 @@ class Scheduler(object): mem_avail = mem_total - mem_used mem_inst = instance_ref['memory_mb'] if mem_avail <= mem_inst: - raise exception.NotEmpty(_("%(ec2_id)s is not capable to " - "migrate %(dest)s (host:%(mem_avail)s " - " <= instance:%(mem_inst)s)") + raise exception.NotEmpty(_("Unable to migrate %(ec2_id)s " + "to destination: %(dest)s " + "(host:%(mem_avail)s " + "<= instance:%(mem_inst)s)") % locals()) def mounted_on_same_shared_storage(self, context, instance_ref, dest): @@ -292,7 +292,7 @@ class Scheduler(object): except rpc.RemoteError: ipath = FLAGS.instances_path - logging.error(_("Cannot comfirm tmpfile at %(ipath)s is on " + logging.error(_("Cannot confirm tmpfile at %(ipath)s is on " "same shared storage between %(src)s " "and %(dest)s.") % locals()) raise diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 85c2c948b..71899ba9e 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -89,14 +89,14 @@ class ComputeTestCase(test.TestCase): Use this when any testcase executed later than test_run_terminate """ vol1 = models.Volume() - vol1.__setitem__('id', 1) + vol1['id'] = 1 vol2 = models.Volume() - vol2.__setitem__('id', 2) + vol2['id'] = 2 instance_ref = models.Instance() - instance_ref.__setitem__('id', 1) - instance_ref.__setitem__('volumes', [vol1, vol2]) - instance_ref.__setitem__('hostname', 'i-00000001') - instance_ref.__setitem__('host', 'dummy') + instance_ref['id'] = 1 + instance_ref['volumes'] = [vol1, vol2] + instance_ref['hostname'] = 'i-00000001' + instance_ref['host'] = 'dummy' return instance_ref def test_create_instance_defaults_display_name(self): @@ -114,9 +114,9 @@ class ComputeTestCase(test.TestCase): """Make sure create associates security groups""" group = self._create_group() instance_ref = models.Instance() - instance_ref.__setitem__('id', 1) - instance_ref.__setitem__('volumes', [{'id': 1}, {'id': 2}]) - instance_ref.__setitem__('hostname', 'i-00000001') + instance_ref['id'] = 1 + instance_ref['volumes'] = [{'id': 1}, {'id': 2}] + instance_ref['hostname'] = 'i-00000001' return instance_ref def test_create_instance_defaults_display_name(self): @@ -390,7 +390,7 @@ class ComputeTestCase(test.TestCase): def test_pre_live_migration_instance_has_no_volume(self): """Confirm log meg when instance doesn't mount any volumes.""" i_ref = self._get_dummy_instance() - i_ref.__setitem__('volumes', []) + i_ref['volumes'] = [] c = context.get_admin_context() self._setup_other_managers() @@ -501,7 +501,7 @@ class ComputeTestCase(test.TestCase): def test_live_migration_dest_raises_exception_no_volume(self): """Same as above test(input pattern is different) """ i_ref = self._get_dummy_instance() - i_ref.__setitem__('volumes', []) + i_ref['volumes'] = [] c = context.get_admin_context() topic = db.queue_get_for(c, FLAGS.compute_topic, i_ref['host']) @@ -526,7 +526,7 @@ class ComputeTestCase(test.TestCase): def test_live_migration_works_correctly_no_volume(self): """Confirm live_migration() works as expected correctly.""" i_ref = self._get_dummy_instance() - i_ref.__setitem__('volumes', []) + i_ref['volumes'] = [] c = context.get_admin_context() topic = db.queue_get_for(c, FLAGS.compute_topic, i_ref['host']) diff --git a/nova/tests/test_scheduler.py b/nova/tests/test_scheduler.py index 711b66af7..8ac02c5a4 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/test_scheduler.py @@ -119,7 +119,8 @@ class SchedulerTestCase(test.TestCase): try: scheduler.show_host_resources(ctxt, dest) except exception.NotFound, e: - c1 = (0 <= e.message.find('does not exist or not compute node')) + c1 = (e.message.find(_("does not exist or is not a " + "compute node.")) >= 0) self.assertTrue(c1) def _dic_is_equal(self, dic1, dic2, keys=None): @@ -786,7 +787,7 @@ class SimpleDriverTestCase(test.TestCase): i_ref, 'somewhere') except exception.NotEmpty, e: - c = (e.message.find('is not capable to migrate') >= 0) + c = (e.message.find('Unable to migrate') >= 0) self.assertTrue(c) db.instance_destroy(self.context, instance_id) diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index d17f6a22a..666c4a11d 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -42,24 +42,6 @@ class FakeManager(manager.Manager): def test_method(self): return 'manager' -# temporary variable to store host/binary/self.mox -# from each method to fake class. -global_host = None -global_binary = None -global_mox = None - - -class FakeComputeManager(compute_manager.ComputeManager): - """Fake computemanager manager for tests""" - - def __init__(self, compute_driver=None, *args, **kwargs): - global ghost, gbinary, gmox - self.update_available_resource(mox.IgnoreArg()) - gmox.ReplayAll() - super(FakeComputeManager, self).__init__(compute_driver, - *args, - **kwargs) - class ExtendedService(service.Service): def test_method(self): @@ -275,37 +257,38 @@ class ServiceTestCase(test.TestCase): """Confirm compute updates their record of compute-service table.""" host = 'foo' binary = 'nova-compute' - topic = 'compute1' - service_create = {'host': host, - 'binary': binary, - 'topic': topic, - 'report_count': 0, - 'availability_zone': 'nova'} - service_ref = {'host': host, - 'binary': binary, - 'topic': topic, - 'report_count': 0, - 'availability_zone': 'nova', - 'id': 1} - - service.db.service_get_by_args(mox.IgnoreArg(), - host, - binary).AndRaise(exception.NotFound()) - service.db.service_create(mox.IgnoreArg(), - service_create).AndReturn(service_ref) - self.mox.StubOutWithMock(compute_manager.ComputeManager, - 'update_available_resource') - - global ghost, gbinary, gmox - ghost = host - gbinary = binary - gmox = self.mox - + topic = 'compute' + + # Any mocks are not working without UnsetStubs() here. + self.mox.UnsetStubs() + ctxt = context.get_admin_context() + service_ref = db.service_create(ctxt, {'host': host, + 'binary': binary, + 'topic': topic}) serv = service.Service(host, binary, topic, - 'nova.tests.test_service.FakeComputeManager') - # ReplayAll has been executed FakeComputeManager.__init__() - #self.mox.ReplayAll() + 'nova.compute.manager.ComputeManager') + + # This testcase want to test calling update_available_resource. + # No need to call periodic call, then below variable must be set 0. + serv.report_interval = 0 + serv.periodic_interval = 0 + + # Creating mocks + self.mox.StubOutWithMock(service.rpc.Connection, 'instance') + service.rpc.Connection.instance(new=mox.IgnoreArg()) + service.rpc.Connection.instance(new=mox.IgnoreArg()) + self.mox.StubOutWithMock(serv.manager.driver, + 'update_available_resource') + serv.manager.driver.update_available_resource(mox.IgnoreArg(), host) + + # Just doing start()-stop(), not confirm new db record is created, + # because update_available_resource() works only in libvirt environment. + # This testcase confirms update_available_resource() is called. + # Otherwise, mox complains. + self.mox.ReplayAll() serv.start() serv.stop() + + db.service_destroy(ctxt, service_ref['id']) diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 7ea8c0fb5..ee41ae732 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -283,7 +283,7 @@ class LibvirtConnTestCase(test.TestCase): self.assertEquals(uri, testuri) db.instance_destroy(user_context, instance_ref['id']) - def tes1t_update_available_resource_works_correctly(self): + def test_update_available_resource_works_correctly(self): """Confirm compute_node table is updated successfully.""" org_path = FLAGS.instances_path = '' FLAGS.instances_path = '.' @@ -314,7 +314,7 @@ class LibvirtConnTestCase(test.TestCase): compute_node = service_ref['compute_node'][0] if sys.platform.upper() == 'LINUX2': - self.assertTrue(compute_node['vcpus'] > 0) + self.assertTrue(compute_node['vcpus'] >= 0) self.assertTrue(compute_node['memory_mb'] > 0) self.assertTrue(compute_node['local_gb'] > 0) self.assertTrue(compute_node['vcpus_used'] == 0) @@ -323,7 +323,7 @@ class LibvirtConnTestCase(test.TestCase): self.assertTrue(len(compute_node['hypervisor_type']) > 0) self.assertTrue(compute_node['hypervisor_version'] > 0) else: - self.assertTrue(compute_node['vcpus'] > 0) + self.assertTrue(compute_node['vcpus'] >= 0) self.assertTrue(compute_node['memory_mb'] == 0) self.assertTrue(compute_node['local_gb'] > 0) self.assertTrue(compute_node['vcpus_used'] == 0) diff --git a/nova/tests/test_volume.py b/nova/tests/test_volume.py index e8b4ceee8..d88e363da 100644 --- a/nova/tests/test_volume.py +++ b/nova/tests/test_volume.py @@ -284,9 +284,10 @@ class AOETestCase(DriverTestCase): self.volume.check_for_export(self.context, self.instance_id) except exception.ProcessExecutionError, e: volume_id = volume_id_list[0] - msg = _("""Cannot confirm exported volume id:%(volume_id)s.""" - """vblade process for e%(shelf_id)s.%(blade_id)s """ - """isn't running.""") % locals() + msg = _("Cannot confirm exported volume id:%(volume_id)s. " + "vblade process for e%(shelf_id)s.%(blade_id)s " + "isn't running.") % locals() + msg_is_match = (0 <= e.message.find(msg)) self.assertTrue(msg_is_match) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 938719a7c..43a9dc4e7 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -860,7 +860,14 @@ class LibvirtConnection(object): """ - return multiprocessing.cpu_count() + # On certain platforms, this will raise a NotImplementedError. + try: + return multiprocessing.cpu_count() + except NotImplementedError: + LOG.warn(_("Cannot get the number of cpu, because this " + "function is not implemented for this platform. " + "This error can be safely ignored for now.")) + return 0 def get_memory_mb_total(self): """Get the total memory size(MB) of physical computer. @@ -1042,9 +1049,9 @@ class LibvirtConnection(object): try: service_ref = db.service_get_all_compute_by_host(ctxt, host)[0] except exception.NotFound: - msg = _(("""Cannot update compute manager specific info,""" - """ Because no service record found.""")) - raise exception.Invalid(msg) + raise exception.Invalid(_("Cannot update compute manager " + "specific info, because no service " + "record was found.")) # Updating host information dic = {'vcpus': self.get_vcpu_total(), @@ -1059,11 +1066,11 @@ class LibvirtConnection(object): compute_node_ref = service_ref['compute_node'] if not compute_node_ref: - LOG.info(_('Compute_service record is created for %s ') % host) + LOG.info(_('Compute_service record created for %s ') % host) dic['service_id'] = service_ref['id'] db.compute_node_create(ctxt, dic) else: - LOG.info(_('Compute_service record is updated for %s ') % host) + LOG.info(_('Compute_service record updated for %s ') % host) db.compute_node_update(ctxt, compute_node_ref[0]['id'], dic) def compare_cpu(self, cpu_info): @@ -1081,8 +1088,7 @@ class LibvirtConnection(object): """ - LOG.info(_('Checking cpu_info: instance was launched this cpu.\n%s') - % cpu_info) + LOG.info(_('Instance launched has CPU info:\n%s') % cpu_info) dic = utils.loads(cpu_info) xml = str(Template(self.cpuinfo_xml, searchList=dic)) LOG.info(_('to xml...\n:%s ' % xml)) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index a902da6ac..31a6a02ee 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -234,7 +234,7 @@ class AOEDriver(VolumeDriver): break if not exported: # Instance will be terminated in this case. - desc = _("Cannot confirm exported volume id:%(volume_id)s." + desc = _("Cannot confirm exported volume id:%(volume_id)s. " "vblade process for e%(shelf_id)s.%(blade_id)s " "isn't running.") % locals() raise exception.ProcessExecutionError(out, _err, cmd=cmd, -- cgit From 90f38451e5df4f0ca862401cf898f01ffede6174 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 10 Mar 2011 00:26:25 -0500 Subject: add tests to verify the serialization of adminPass in server creation response --- nova/tests/api/openstack/test_servers.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 2fc28fe67..0561ad499 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -19,6 +19,7 @@ import base64 import datetime import json import unittest +from xml.dom import minidom import stubout import webob @@ -908,6 +909,7 @@ class TestServerInstanceCreation(test.TestCase): def _get_create_request_xml(self, body_dict): req = webob.Request.blank('/v1.0/servers') req.content_type = 'application/xml' + req.accept = 'application/xml' req.method = 'POST' req.body = self._format_xml_request_body(body_dict) return req @@ -1034,6 +1036,23 @@ class TestServerInstanceCreation(test.TestCase): self.assertEquals(response.status_int, 200) self.assertEquals(personality_files, [(path, contents)]) + def test_create_instance_admin_pass_json(self): + request, response, dummy = \ + self._create_instance_with_personality_json(None) + self.assertEquals(response.status_int, 200) + response = json.loads(response.body) + self.assertTrue('adminPass' in response['server']) + self.assertTrue(response['server']['adminPass'].startswith('fake')) + + def test_create_instance_admin_pass_xml(self): + request, response, dummy = \ + self._create_instance_with_personality_xml(None) + self.assertEquals(response.status_int, 200) + dom = minidom.parseString(response.body) + server = dom.childNodes[0] + self.assertEquals(server.nodeName, 'server') + self.assertTrue(server.getAttribute('adminPass').startswith('fake')) + if __name__ == "__main__": unittest.main() -- cgit From b75ab789194f1ced801b1d68ae8cc54051716414 Mon Sep 17 00:00:00 2001 From: Kei Masumoto Date: Thu, 10 Mar 2011 15:16:03 +0900 Subject: fix pep8 check --- nova/tests/test_service.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index 666c4a11d..393f9d20b 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -284,9 +284,9 @@ class ServiceTestCase(test.TestCase): serv.manager.driver.update_available_resource(mox.IgnoreArg(), host) # Just doing start()-stop(), not confirm new db record is created, - # because update_available_resource() works only in libvirt environment. - # This testcase confirms update_available_resource() is called. - # Otherwise, mox complains. + # because update_available_resource() works only in + # libvirt environment. This testcase confirms + # update_available_resource() is called. Otherwise, mox complains. self.mox.ReplayAll() serv.start() serv.stop() -- cgit From e76aad24ce8a9b1b7de1b2f874c22c9995f3071f Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 14:30:17 +0100 Subject: Only include ramdisk and kernel id if they are actually set. --- nova/api/ec2/cloud.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index b1917e9ea..1d2254225 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -147,8 +147,6 @@ class CloudController(object): instance_ref['id']) ec2_id = ec2utils.id_to_ec2_id(instance_ref['id']) image_ec2_id = self._image_ec2_id(instance_ref['image_id'], 'machine') - k_ec2_id = self._image_ec2_id(instance_ref['kernel_id'], 'kernel') - r_ec2_id = self._image_ec2_id(instance_ref['ramdisk_id'], 'ramdisk') data = { 'user-data': base64.b64decode(instance_ref['user_data']), 'meta-data': { @@ -167,8 +165,6 @@ class CloudController(object): 'instance-type': instance_ref['instance_type'], 'local-hostname': hostname, 'local-ipv4': address, - 'kernel-id': k_ec2_id, - 'ramdisk-id': r_ec2_id, 'placement': {'availability-zone': availability_zone}, 'public-hostname': hostname, 'public-ipv4': floating_ip or '', @@ -176,6 +172,13 @@ class CloudController(object): 'reservation-id': instance_ref['reservation_id'], 'security-groups': '', 'mpi': mpi}} + + for image_type in ['kernel', 'ramdisk']: + if '%s_id' % image_type in instance_ref: + ec2_id = self._image_ec2_id(instance_ref['%s_id' % image_type], + image_type) + data['meta-data']['%s-id' % image_type] = ec2_id + if False: # TODO(vish): store ancestor ids data['ancestor-ami-ids'] = [] if False: # TODO(vish): store product codes -- cgit From 616723fe4e7d52b0b8ddafda10fcfe07a87609c8 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 10 Mar 2011 14:53:13 -0500 Subject: add docstring --- nova/compute/api.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nova/compute/api.py b/nova/compute/api.py index 2766ddc9c..efa051d10 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -81,6 +81,11 @@ class API(base.Base): {"method": "get_network_topic", "args": {'fake': 1}}) def _check_personality_file_quota(self, context, personality_files): + """ + Enforce quota limits on personality files + + Raises a QuotaError if any limit is exceeded + """ limit = quota.allowed_personality_files(context) if len(personality_files) > limit: raise quota.QuotaError(_("Personality limit exceeded. You can " -- cgit From c177074649055f1da2ca97eb3c07139571d4a664 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Thu, 10 Mar 2011 12:10:49 -0800 Subject: Implements basic OpenStack API client, ready to support API tests --- nova/tests/integrated/__init__.py | 20 +++ nova/tests/integrated/api/__init__.py | 20 +++ nova/tests/integrated/api/client.py | 224 ++++++++++++++++++++++++++++++++++ 3 files changed, 264 insertions(+) create mode 100644 nova/tests/integrated/__init__.py create mode 100644 nova/tests/integrated/api/__init__.py create mode 100644 nova/tests/integrated/api/client.py diff --git a/nova/tests/integrated/__init__.py b/nova/tests/integrated/__init__.py new file mode 100644 index 000000000..10e0a91d7 --- /dev/null +++ b/nova/tests/integrated/__init__.py @@ -0,0 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +:mod:`integrated` -- Tests whole systems, using mock services where needed +================================= +""" diff --git a/nova/tests/integrated/api/__init__.py b/nova/tests/integrated/api/__init__.py new file mode 100644 index 000000000..5798ab3d1 --- /dev/null +++ b/nova/tests/integrated/api/__init__.py @@ -0,0 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +:mod:`api` -- OpenStack API client, for testing rather than production +================================= +""" diff --git a/nova/tests/integrated/api/client.py b/nova/tests/integrated/api/client.py new file mode 100644 index 000000000..5ab247fab --- /dev/null +++ b/nova/tests/integrated/api/client.py @@ -0,0 +1,224 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import httplib +import urlparse + +from nova import log as logging + + +LOG = logging.getLogger('nova.tests.api') + + +class OpenstackApiException(Exception): + def __init__(self, message=None, response=None): + self.response = response + if not message: + message = 'Unspecified error' + + if response: + _status = response.status + _body = response.read() + + message = _('%(message)s\nStatus Code: %(_status)s\n' + 'Body: %(_body)s') % locals() + + super(OpenstackApiException, self).__init__(message) + + +class OpenstackApiAuthenticationException(OpenstackApiException): + def __init__(self, response=None, message=None): + if not message: + message = _("Authentication error") + super(OpenstackApiAuthenticationException, self).__init__(message, + response) + + +class OpenstackApiNotFoundException(OpenstackApiException): + def __init__(self, response=None, message=None): + if not message: + message = _("Item not found") + super(OpenstackApiNotFoundException, self).__init__(message, response) + + +class TestOpenStackClient(object): + """ A really basic OpenStack API client that is under our control, + so we can make changes / insert hooks for testing""" + + def __init__(self, auth_user, auth_key, auth_uri): + super(TestOpenStackClient, self).__init__() + self.auth_result = None + self.auth_user = auth_user + self.auth_key = auth_key + self.auth_uri = auth_uri + + def request(self, url, method='GET', body=None, headers=None): + if headers is None: + headers = {} + + parsed_url = urlparse.urlparse(url) + port = parsed_url.port + hostname = parsed_url.hostname + scheme = parsed_url.scheme + + if scheme == 'http': + conn = httplib.HTTPConnection(hostname, + port=port) + elif scheme == 'https': + conn = httplib.HTTPSConnection(hostname, + port=port) + else: + raise OpenstackApiException("Unknown scheme: %s" % url) + + relative_url = parsed_url.path + if parsed_url.query: + relative_url = relative_url + parsed_url.query + LOG.info(_("Doing %(method)s on %(relative_url)s") % locals()) + if body: + LOG.info(_("Body: %s") % body) + + conn.request(method, relative_url, body, headers) + response = conn.getresponse() + return response + + def _authenticate(self): + if self.auth_result: + return self.auth_result + + headers = {'X-Auth-User': self.auth_user, + 'X-Auth-Key': self.auth_key} + response = self.request(self.auth_uri, + headers=headers) + if not response.status in [204]: + raise OpenstackApiAuthenticationException(response=response) + + auth_headers = {} + for k, v in response.getheaders(): + auth_headers[k] = v + + self.auth_result = auth_headers + return self.auth_result + + def api_request(self, relative_uri, check_response_status=None, **kwargs): + auth_result = self._authenticate() + + base_uri = auth_result['X-Server-Management-Url'] + full_uri = base_uri + relative_uri + + headers = kwargs.setdefault('headers', {}) + headers['X-Auth-Token'] = auth_result['X-Auth-Token'] + + LOG.debug(_("HTTP request on %s") % (relative_uri)) + + response = self.request(full_uri, **kwargs) + + LOG.debug(_("Response => code %s") % (response.status)) + + if check_response_status: + if not response.status in check_response_status: + if response.status == 404: + raise OpenstackApiNotFoundException(response=response) + else: + raise OpenstackApiException( + message=_("Unexpected status code"), + response=response) + + return response + + def _decode_json(self, response): + body = response.read() + LOG.debug(_("Decoding JSON: %s") % (body)) + return json.loads(body) + + def api_get(self, relative_uri, **kwargs): + kwargs.setdefault('check_response_status', [200]) + response = self.api_request(relative_uri, **kwargs) + return self._decode_json(response) + + def api_post(self, relative_uri, body, **kwargs): + kwargs['method'] = 'POST' + if body: + headers = kwargs.setdefault('headers', {}) + headers['Content-Type'] = 'application/json' + kwargs['body'] = json.dumps(body) + + kwargs.setdefault('check_response_status', [200]) + response = self.api_request(relative_uri, **kwargs) + return self._decode_json(response) + + def api_delete(self, relative_uri, **kwargs): + kwargs['method'] = 'DELETE' + kwargs.setdefault('check_response_status', [200, 202]) + return self.api_request(relative_uri, **kwargs) + + def get_keys_detail(self): + return self.api_get('/keys/detail')['keys'] + + def post_key(self, key): + return self.api_post('/keys', key)['key'] + + def delete_key(self, key_id): + return self.api_delete('/keys/%s' % key_id) + + def get_volume(self, volume_id): + return self.api_get('/volumes/%s' % volume_id)['volume'] + + def get_volumes_detail(self): + return self.api_get('/volumes/detail')['volumes'] + + def post_volume(self, volume): + return self.api_post('/volumes', volume)['volume'] + + def delete_volume(self, volume_id): + return self.api_delete('/volumes/%s' % volume_id) + + def get_server(self, server_id): + return self.api_get('/servers/%s' % server_id)['server'] + + def get_servers(self, detail=True): + rel_url = '/servers/detail' if detail else '/servers' + return self.api_get(rel_url)['servers'] + + def post_server(self, server): + return self.api_post('/servers', server)['server'] + + def delete_server(self, server_id): + return self.api_delete('/servers/%s' % server_id) + + def get_image(self, image_id): + return self.api_get('/images/%s' % image_id)['image'] + + def get_images_detail(self): + return self.api_get('/images/detail')['images'] + + def post_image(self, image): + return self.api_post('/images', image)['image'] + + def delete_image(self, image_id): + return self.api_delete('/images/%s' % image_id) + + def get_flavor(self, flavor_id): + return self.api_get('/flavors/%s' % flavor_id)['flavor'] + + def get_flavors_detail(self): + return self.api_get('/flavors/detail')['flavors'] + + def post_flavor(self, flavor): + return self.api_post('/flavors', flavor)['flavor'] + + def delete_flavor(self, flavor_id): + return self.api_delete('/flavors/%s' % flavor_id) -- cgit From 8aabc32a69bf47075a3fd8e677d1bd70cbbca339 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 21:13:07 +0100 Subject: Add basic test case. --- nova/tests/test_cloud.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index cf8ee7eff..db7c15aeb 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -353,6 +353,18 @@ class CloudTestCase(test.TestCase): self.assertEqual('', img.metadata['description']) shutil.rmtree(pathdir) + def test_metadata_works_without_kernel_and_ramdisk(self): + inst = db.instance_create(self.context, {'host': self.compute.host, + 'vcpus': 2, + 'image_id': '123456', + 'user_data': '' }) + fixed = self.network.allocate_fixed_ip(self.context, inst['id']) + try: + self.cloud.get_metadata(fixed) + finally: + self.network.deallocate_fixed_ip(self.context, fixed) + db.instance_destroy(self.context, inst['id']) + def test_update_of_instance_display_fields(self): inst = db.instance_create(self.context, {}) ec2_id = ec2utils.id_to_ec2_id(inst['id']) -- cgit From 6b4beef8093b4a7b4d42818567b5afb023af9251 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Thu, 10 Mar 2011 12:55:06 -0800 Subject: Don't wrap keys and volumes till they're in the API --- nova/tests/integrated/api/client.py | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/nova/tests/integrated/api/client.py b/nova/tests/integrated/api/client.py index 5ab247fab..8ec46b5ae 100644 --- a/nova/tests/integrated/api/client.py +++ b/nova/tests/integrated/api/client.py @@ -165,27 +165,6 @@ class TestOpenStackClient(object): kwargs.setdefault('check_response_status', [200, 202]) return self.api_request(relative_uri, **kwargs) - def get_keys_detail(self): - return self.api_get('/keys/detail')['keys'] - - def post_key(self, key): - return self.api_post('/keys', key)['key'] - - def delete_key(self, key_id): - return self.api_delete('/keys/%s' % key_id) - - def get_volume(self, volume_id): - return self.api_get('/volumes/%s' % volume_id)['volume'] - - def get_volumes_detail(self): - return self.api_get('/volumes/detail')['volumes'] - - def post_volume(self, volume): - return self.api_post('/volumes', volume)['volume'] - - def delete_volume(self, volume_id): - return self.api_delete('/volumes/%s' % volume_id) - def get_server(self, server_id): return self.api_get('/servers/%s' % server_id)['server'] -- cgit From f81d925f86670e3ed32d815c219824f627d82bc2 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Thu, 10 Mar 2011 13:51:26 -0800 Subject: Better logging, be more careful about when we throw login errors re bug732866 --- nova/tests/integrated/api/client.py | 37 +++++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 14 deletions(-) diff --git a/nova/tests/integrated/api/client.py b/nova/tests/integrated/api/client.py index 8ec46b5ae..d424a6428 100644 --- a/nova/tests/integrated/api/client.py +++ b/nova/tests/integrated/api/client.py @@ -99,11 +99,18 @@ class TestOpenStackClient(object): if self.auth_result: return self.auth_result + auth_uri = self.auth_uri headers = {'X-Auth-User': self.auth_user, 'X-Auth-Key': self.auth_key} - response = self.request(self.auth_uri, + response = self.request(auth_uri, headers=headers) - if not response.status in [204]: + + http_status = response.status + LOG.debug(_("%(auth_uri)s => code %(http_status)s") % locals()) + + # Until bug732866 is fixed, we can't check this properly... + #if http_status == 401: + if http_status != 204: raise OpenstackApiAuthenticationException(response=response) auth_headers = {} @@ -116,21 +123,21 @@ class TestOpenStackClient(object): def api_request(self, relative_uri, check_response_status=None, **kwargs): auth_result = self._authenticate() - base_uri = auth_result['X-Server-Management-Url'] + #NOTE(justinsb): httplib 'helpfully' converts headers to lower case + base_uri = auth_result['x-server-management-url'] full_uri = base_uri + relative_uri headers = kwargs.setdefault('headers', {}) - headers['X-Auth-Token'] = auth_result['X-Auth-Token'] - - LOG.debug(_("HTTP request on %s") % (relative_uri)) + headers['X-Auth-Token'] = auth_result['x-auth-token'] response = self.request(full_uri, **kwargs) - - LOG.debug(_("Response => code %s") % (response.status)) + + http_status = response.status + LOG.debug(_("%(relative_uri)s => code %(http_status)s") % locals()) if check_response_status: - if not response.status in check_response_status: - if response.status == 404: + if not http_status in check_response_status: + if http_status == 404: raise OpenstackApiNotFoundException(response=response) else: raise OpenstackApiException( @@ -181,8 +188,9 @@ class TestOpenStackClient(object): def get_image(self, image_id): return self.api_get('/images/%s' % image_id)['image'] - def get_images_detail(self): - return self.api_get('/images/detail')['images'] + def get_images(self, detail=True): + rel_url = '/images/detail' if detail else '/images' + return self.api_get(rel_url)['images'] def post_image(self, image): return self.api_post('/images', image)['image'] @@ -193,8 +201,9 @@ class TestOpenStackClient(object): def get_flavor(self, flavor_id): return self.api_get('/flavors/%s' % flavor_id)['flavor'] - def get_flavors_detail(self): - return self.api_get('/flavors/detail')['flavors'] + def get_flavors(self, detail=True): + rel_url = '/flavors/detail' if detail else '/flavors' + return self.api_get(rel_url)['flavors'] def post_flavor(self, flavor): return self.api_post('/flavors', flavor)['flavor'] -- cgit From f110e718807ea4747a0ff95138c488961257aa7f Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Thu, 10 Mar 2011 13:56:24 -0800 Subject: pep8 fun --- nova/tests/integrated/api/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/integrated/api/client.py b/nova/tests/integrated/api/client.py index d424a6428..da8d87e07 100644 --- a/nova/tests/integrated/api/client.py +++ b/nova/tests/integrated/api/client.py @@ -104,7 +104,7 @@ class TestOpenStackClient(object): 'X-Auth-Key': self.auth_key} response = self.request(auth_uri, headers=headers) - + http_status = response.status LOG.debug(_("%(auth_uri)s => code %(http_status)s") % locals()) @@ -131,7 +131,7 @@ class TestOpenStackClient(object): headers['X-Auth-Token'] = auth_result['x-auth-token'] response = self.request(full_uri, **kwargs) - + http_status = response.status LOG.debug(_("%(relative_uri)s => code %(http_status)s") % locals()) -- cgit From c967679fa8144af57d79d89666ee29a0241d38a9 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Thu, 10 Mar 2011 17:36:41 -0500 Subject: switch to a more consistent usage of onset_files variable names --- nova/api/openstack/servers.py | 59 ++++++++++++++++++++----------- nova/compute/api.py | 38 ++++++++------------ nova/compute/manager.py | 2 +- nova/quota.py | 30 ++++++++-------- nova/tests/api/openstack/test_servers.py | 48 ++++++++++++------------- nova/tests/test_quota.py | 60 ++++++++++++++++---------------- 6 files changed, 124 insertions(+), 113 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 7bef8eb32..adb5c5f99 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -169,20 +169,23 @@ class Controller(wsgi.Controller): metadata.append({'key': k, 'value': v}) personality = env['server'].get('personality', []) - personality_files = self._get_personality_files(personality) - - instances = self.compute_api.create( - context, - instance_types.get_by_flavor_id(env['server']['flavorId']), - image_id, - kernel_id=kernel_id, - ramdisk_id=ramdisk_id, - display_name=env['server']['name'], - display_description=env['server']['name'], - key_name=key_pair['name'], - key_data=key_pair['public_key'], - metadata=metadata, - personality_files=personality_files) + onset_files = self._get_onset_files(personality) + + try: + instances = self.compute_api.create( + context, + instance_types.get_by_flavor_id(env['server']['flavorId']), + image_id, + kernel_id=kernel_id, + ramdisk_id=ramdisk_id, + display_name=env['server']['name'], + display_description=env['server']['name'], + key_name=key_pair['name'], + key_data=key_pair['public_key'], + metadata=metadata, + onset_files=onset_files) + except QuotaError as error: + self._handle_quota_error(error) server = _translate_keys(instances[0]) password = "%s%s" % (server['server']['name'][:4], @@ -204,15 +207,15 @@ class Controller(wsgi.Controller): else: return self._deserialize(request.body, request.get_content_type()) - def _get_personality_files(self, personality): + def _get_onset_files(self, personality): """ - Create a list of personality files from the personality attribute + Create a list of onset files from the personality attribute - At this time, personality_files must be formatted as a list of + At this time, onset_files must be formatted as a list of (file_path, file_content) pairs for compatibility with the underlying compute service. """ - personality_files = [] + onset_files = [] for item in personality: try: path = item['path'] @@ -227,8 +230,24 @@ class Controller(wsgi.Controller): except TypeError: msg = 'Personality content for %s cannot be decoded' % path raise exc.HTTPBadRequest(explanation=msg) - personality_files.append((path, contents)) - return personality_files + onset_files.append((path, contents)) + return onset_files + + def _handle_quota_errors(self, error): + """ + Reraise quota errors as api-specific http exceptions + """ + if error.code == "OnsetFileLimitExceeded": + expl = "Personality file limit exceeded" + raise exc.HTTPBadRequest(explanation=expl) + if error.code == "OnsetFilePathLimitExceeded": + expl = "Personality file path too long" + raise exc.HTTPBadRequest(explanation=expl) + if error.code == "OnsetFileContentLimitExceeded": + expl = "Personality file content too long" + raise exc.HTTPBadRequest(explanation=expl) + # if the original error is okay, just reraise it + raise error def update(self, req, id): """ Updates the server name or password """ diff --git a/nova/compute/api.py b/nova/compute/api.py index b97cadf61..140bbb3aa 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -80,30 +80,23 @@ class API(base.Base): topic, {"method": "get_network_topic", "args": {'fake': 1}}) - def _check_personality_file_quota(self, context, personality_files): + def _check_onset_file_quota(self, context, onset_files): """ - Enforce quota limits on personality files + Enforce quota limits on onset files Raises a QuotaError if any limit is exceeded """ - limit = quota.allowed_personality_files(context) - if len(personality_files) > limit: - raise quota.QuotaError(_("Personality limit exceeded. You can " - "only have %d personalities when " - "creating an instance.") % limit, - "PersonalityLimitExceeded") - path_limit = quota.allowed_personality_path_bytes(context) - content_limit = quota.allowed_personality_content_bytes(context) - for path, content in personality_files: + limit = quota.allowed_onset_files(context) + if len(onset_files) > limit: + raise quota.QuotaError(code="OnsetFileLimitExceeded") + path_limit = quota.allowed_onset_file_path_bytes(context) + content_limit = quota.allowed_onset_file_content_bytes(context) + for path, content in onset_files: if len(path) > path_limit: - raise quota.QuotaError( - _("Personality file path limit exceeded."), - "PersonalityLimitExceeded") + raise quota.QuotaError(code="OnsetFilePathLimitExceeded") if len(content) > content_limit: - raise quota.QuotaError( - _("Personality file content limit exceeded."), - "PersonalityLimitExceeded") - return personality_files + raise quota.QuotaError(code="OnsetFileContentLimitExceeded") + return onset_files def create(self, context, instance_type, image_id, kernel_id=None, ramdisk_id=None, @@ -111,7 +104,7 @@ class API(base.Base): display_name='', display_description='', key_name=None, key_data=None, security_group='default', availability_zone=None, user_data=None, metadata=[], - personality_files=None): + onset_files=None): """Create the number of instances requested if quota and other arguments check out ok.""" @@ -149,9 +142,8 @@ class API(base.Base): LOG.warn(msg) raise quota.QuotaError(msg, "MetadataLimitExceeded") - if personality_files is not None: - personality_files = \ - self._check_personality_file_quota(context, personality_files) + if onset_files is not None: + onset_files = self._check_onset_file_quota(context, onset_files) image = self.image_service.show(context, image_id) if kernel_id is None: @@ -248,7 +240,7 @@ class API(base.Base): "args": {"topic": FLAGS.compute_topic, "instance_id": instance_id, "availability_zone": availability_zone, - "personality_files": personality_files}}) + "onset_files": onset_files}}) for group_id in security_groups: self.trigger_security_group_members_refresh(elevated, group_id) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index d87290aae..601bb3084 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -173,7 +173,7 @@ class ComputeManager(manager.Manager): """Launch a new instance with specified options.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) - instance_ref.onset_files = kwargs.get('personality_files', []) + instance_ref.onset_files = kwargs.get('onset_files', []) if instance_ref['name'] in self.driver.list_instances(): raise exception.Error(_("Instance has already been created")) LOG.audit(_("instance %s: starting..."), instance_id, diff --git a/nova/quota.py b/nova/quota.py index fdef42bc5..e0fb97542 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -37,12 +37,12 @@ flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') flags.DEFINE_integer('quota_metadata_items', 128, 'number of metadata items allowed per instance') -flags.DEFINE_integer('quota_personality_max_files', 5, - 'number of personality files allowed') -flags.DEFINE_integer('quota_personality_max_content_bytes', 10 * 1024, - 'number of bytes allowed per personality file') -flags.DEFINE_integer('quota_personality_max_path_bytes', 255, - 'number of bytes allowed per personality file path') +flags.DEFINE_integer('quota_max_onset_files', 5, + 'number of onset files allowed') +flags.DEFINE_integer('quota_max_onset_file_content_bytes', 10 * 1024, + 'number of bytes allowed per onset file') +flags.DEFINE_integer('quota_max_onset_file_path_bytes', 255, + 'number of bytes allowed per onset file path') def get_quota(context, project_id): @@ -113,19 +113,19 @@ def allowed_metadata_items(context, num_metadata_items): return min(num_metadata_items, num_allowed_metadata_items) -def allowed_personality_files(context): - """Return the number of personality files allowed""" - return int(FLAGS.quota_personality_max_files) +def allowed_onset_files(context): + """Return the number of onset files allowed""" + return int(FLAGS.quota_max_onset_files) -def allowed_personality_content_bytes(context): - """Return the number of bytes allowed per personality content""" - return int(FLAGS.quota_personality_max_content_bytes) +def allowed_onset_file_content_bytes(context): + """Return the number of bytes allowed per onset file content""" + return int(FLAGS.quota_max_onset_file_content_bytes) -def allowed_personality_path_bytes(context): - """Return the number of bytes allowed in a personality file path""" - return int(FLAGS.quota_personality_max_path_bytes) +def allowed_onset_file_path_bytes(context): + """Return the number of bytes allowed in an onset file path""" + return int(FLAGS.quota_max_onset_file_path_bytes) class QuotaError(exception.ApiError): diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 0561ad499..b6d88d833 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -833,13 +833,13 @@ class TestServerInstanceCreation(test.TestCase): class MockComputeAPI(object): def __init__(self): - self.personality_files = None + self.onset_files = None def create(self, *args, **kwargs): - if 'personality_files' in kwargs: - self.personality_files = kwargs['personality_files'] + if 'onset_files' in kwargs: + self.onset_files = kwargs['onset_files'] else: - self.personality_files = None + self.onset_files = None return [{'id': '1234', 'display_name': 'fakeinstance'}] def set_admin_password(self, *args, **kwargs): @@ -919,46 +919,46 @@ class TestServerInstanceCreation(test.TestCase): request = self._get_create_request_json(body_dict) compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) - return request, response, compute_api.personality_files + return request, response, compute_api.onset_files def _create_instance_with_personality_xml(self, personality): body_dict = self._create_personality_request_dict(personality) request = self._get_create_request_xml(body_dict) compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) - return request, response, compute_api.personality_files + return request, response, compute_api.onset_files def test_create_instance_with_no_personality(self): - request, response, personality_files = \ + request, response, onset_files = \ self._create_instance_with_personality_json(personality=None) self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, []) + self.assertEquals(onset_files, []) def test_create_instance_with_no_personality_xml(self): - request, response, personality_files = \ + request, response, onset_files = \ self._create_instance_with_personality_xml(personality=None) self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, []) + self.assertEquals(onset_files, []) def test_create_instance_with_personality(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Hello, World!"\n' b64contents = base64.b64encode(contents) personality = [(path, b64contents)] - request, response, personality_files = \ + request, response, onset_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, [(path, contents)]) + self.assertEquals(onset_files, [(path, contents)]) def test_create_instance_with_personality_xml(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Hello, World!"\n' b64contents = base64.b64encode(contents) personality = [(path, b64contents)] - request, response, personality_files = \ + request, response, onset_files = \ self._create_instance_with_personality_xml(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, [(path, contents)]) + self.assertEquals(onset_files, [(path, contents)]) def test_create_instance_with_personality_no_path(self): personality = [('/remove/this/path', @@ -969,7 +969,7 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) + self.assertEquals(compute_api.onset_files, None) def _test_create_instance_with_personality_no_path_xml(self): personality = [('/remove/this/path', @@ -980,7 +980,7 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) + self.assertEquals(compute_api.onset_files, None) def test_create_instance_with_personality_no_contents(self): personality = [('/test/path', @@ -991,7 +991,7 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) + self.assertEquals(compute_api.onset_files, None) def test_create_instance_with_personality_not_a_list(self): personality = [('/test/path', base64.b64encode('test\ncontents\n'))] @@ -1002,16 +1002,16 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.personality_files, None) + self.assertEquals(compute_api.onset_files, None) def test_create_instance_with_personality_with_non_b64_content(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Oh no!"\n' personality = [(path, contents)] - request, response, personality_files = \ + request, response, onset_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 400) - self.assertEquals(personality_files, None) + self.assertEquals(onset_files, None) def test_create_instance_with_three_personalities(self): files = [ @@ -1022,19 +1022,19 @@ class TestServerInstanceCreation(test.TestCase): personality = [] for path, content in files: personality.append((path, base64.b64encode(content))) - request, response, personality_files = \ + request, response, onset_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, files) + self.assertEquals(onset_files, files) def test_create_instance_personality_empty_content(self): path = '/my/file/path' contents = '' personality = [(path, contents)] - request, response, personality_files = \ + request, response, onset_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(personality_files, [(path, contents)]) + self.assertEquals(onset_files, [(path, contents)]) def test_create_instance_admin_pass_json(self): request, response, dummy = \ diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 86b1d40fb..d94381aa2 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -200,66 +200,66 @@ class QuotaTestCase(test.TestCase): image_id='fake', metadata=metadata) - def test_allowed_personality_files(self): + def test_allowed_onset_files(self): self.assertEqual( - quota.allowed_personality_files(self.context), - FLAGS.quota_personality_max_files) + quota.allowed_onset_files(self.context), + FLAGS.quota_max_onset_files) - def _create_with_personality(self, files): + def _create_with_onset_files(self, files): api = compute.API(image_service=self.StubImageService()) api.create(self.context, min_count=1, max_count=1, instance_type='m1.small', image_id='fake', - personality_files=files) + onset_files=files) - def test_no_personality_files(self): + def test_no_onset_files(self): api = compute.API(image_service=self.StubImageService()) api.create(self.context, instance_type='m1.small', image_id='fake') - def test_max_personality_files(self): + def test_max_onset_files(self): files = [] - for i in xrange(FLAGS.quota_personality_max_files): + for i in xrange(FLAGS.quota_max_onset_files): files.append(('/my/path%d' % i, 'config = test\n')) - self._create_with_personality(files) # no QuotaError + self._create_with_onset_files(files) # no QuotaError - def test_too_many_personality_files(self): + def test_too_many_onset_files(self): files = [] - for i in xrange(FLAGS.quota_personality_max_files + 1): + for i in xrange(FLAGS.quota_max_onset_files + 1): files.append(('/my/path%d' % i, 'my\ncontent%d\n' % i)) self.assertRaises(quota.QuotaError, - self._create_with_personality, files) + self._create_with_onset_files, files) - def test_allowed_personality_content_bytes(self): + def test_allowed_onset_file_content_bytes(self): self.assertEqual( - quota.allowed_personality_content_bytes(self.context), - FLAGS.quota_personality_max_content_bytes) + quota.allowed_onset_file_content_bytes(self.context), + FLAGS.quota_max_onset_file_content_bytes) - def test_max_personality_content_bytes(self): - max = FLAGS.quota_personality_max_content_bytes + def test_max_onset_file_content_bytes(self): + max = FLAGS.quota_max_onset_file_content_bytes content = ''.join(['a' for i in xrange(max)]) files = [('/test/path', content)] - self._create_with_personality(files) # no QuotaError + self._create_with_onset_files(files) # no QuotaError - def test_too_many_personality_content_bytes(self): - max = FLAGS.quota_personality_max_content_bytes + def test_too_many_onset_file_content_bytes(self): + max = FLAGS.quota_max_onset_file_content_bytes content = ''.join(['a' for i in xrange(max + 1)]) files = [('/test/path', content)] self.assertRaises(quota.QuotaError, - self._create_with_personality, files) + self._create_with_onset_files, files) - def test_allowed_personality_path_bytes(self): + def test_allowed_onset_file_path_bytes(self): self.assertEqual( - quota.allowed_personality_path_bytes(self.context), - FLAGS.quota_personality_max_path_bytes) + quota.allowed_onset_file_path_bytes(self.context), + FLAGS.quota_max_onset_file_path_bytes) - def test_max_personality_path_bytes(self): - max = FLAGS.quota_personality_max_path_bytes + def test_max_onset_file_path_bytes(self): + max = FLAGS.quota_max_onset_file_path_bytes path = ''.join(['a' for i in xrange(max)]) files = [(path, 'config = quotatest')] - self._create_with_personality(files) # no QuotaError + self._create_with_onset_files(files) # no QuotaError - def test_too_many_personality_path_bytes(self): - max = FLAGS.quota_personality_max_path_bytes + def test_too_many_onset_file_path_bytes(self): + max = FLAGS.quota_max_onset_file_path_bytes path = ''.join(['a' for i in xrange(max + 1)]) files = [(path, 'config = quotatest')] self.assertRaises(quota.QuotaError, - self._create_with_personality, files) + self._create_with_onset_files, files) -- cgit From 67a8d635af0a64ad220b163c00b96eadf7daf93f Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 11 Mar 2011 09:54:08 +0100 Subject: Make Authors check account for tests being run with different os.getcwd() depending on how they're run. Add missing people to Authors. --- .mailmap | 1 + Authors | 1 + nova/tests/test_misc.py | 12 +++++++----- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.mailmap b/.mailmap index ed4404ad5..ccf2109a7 100644 --- a/.mailmap +++ b/.mailmap @@ -28,6 +28,7 @@ + diff --git a/Authors b/Authors index 7993955e2..4ee0643cf 100644 --- a/Authors +++ b/Authors @@ -19,6 +19,7 @@ Devin Carlen Ed Leafe Eldar Nugaev Eric Day +Eric Windisch Ewan Mellor Hisaharu Ishii Hisaki Ohara diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index a658e4978..1fbaf304f 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -24,18 +24,19 @@ from nova.utils import parse_mailmap, str_dict_replace, synchronized class ProjectTestCase(test.TestCase): def test_authors_up_to_date(self): - if os.path.exists('.bzr'): + topdir = os.path.normpath(os.path.dirname(__file__) + '/../../') + if os.path.exists(os.path.join(topdir, '.bzr')): contributors = set() - mailmap = parse_mailmap('.mailmap') + mailmap = parse_mailmap(os.path.join(topdir, '.mailmap')) import bzrlib.workingtree - tree = bzrlib.workingtree.WorkingTree.open('.') + tree = bzrlib.workingtree.WorkingTree.open(topdir) tree.lock_read() try: parents = tree.get_parent_ids() g = tree.branch.repository.get_graph() - for p in parents[1:]: + for p in parents: rev_ids = [r for r, _ in g.iter_ancestry(parents) if r != "null:"] revs = tree.branch.repository.get_revisions(rev_ids) @@ -44,7 +45,8 @@ class ProjectTestCase(test.TestCase): email = author.split(' ')[-1] contributors.add(str_dict_replace(email, mailmap)) - authors_file = open('Authors', 'r').read() + authors_file = open(os.path.join(topdir, 'Authors'), + 'r').read() missing = set() for contributor in contributors: -- cgit From d250d522b5d6c164435fc254223ff9a0f055cf05 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 11 Mar 2011 17:55:28 +0100 Subject: Remove broken test. At least this way, it'll actually fix the problem and be mergable. --- nova/tests/test_cloud.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index db7c15aeb..cf8ee7eff 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -353,18 +353,6 @@ class CloudTestCase(test.TestCase): self.assertEqual('', img.metadata['description']) shutil.rmtree(pathdir) - def test_metadata_works_without_kernel_and_ramdisk(self): - inst = db.instance_create(self.context, {'host': self.compute.host, - 'vcpus': 2, - 'image_id': '123456', - 'user_data': '' }) - fixed = self.network.allocate_fixed_ip(self.context, inst['id']) - try: - self.cloud.get_metadata(fixed) - finally: - self.network.deallocate_fixed_ip(self.context, fixed) - db.instance_destroy(self.context, inst['id']) - def test_update_of_instance_display_fields(self): inst = db.instance_create(self.context, {}) ec2_id = ec2utils.id_to_ec2_id(inst['id']) -- cgit From be9734b03bce871d32e21da2ba341dfa42aa020a Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Fri, 11 Mar 2011 17:19:14 -0500 Subject: Fixed lp732866 by catching relevant `exception.NotFound` exception. Tests did not uncover this vulnerability due to "incorrect" FakeAuthManager. I say "incorrect" because potentially different implementations (LDAP or Database driven) of AuthManager might return different errors from `get_user_from_access_key`. Also, removed all references to 'bacon', 'ham', 'herp', and 'derp' and replaced them with hopefully more helpful terms. --- nova/api/openstack/auth.py | 6 +++- nova/tests/api/openstack/fakes.py | 5 +++- nova/tests/api/openstack/test_auth.py | 56 +++++++++++++++++++++-------------- 3 files changed, 43 insertions(+), 24 deletions(-) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 4c6b58eff..f3a9bdeca 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -135,7 +135,11 @@ class AuthMiddleware(wsgi.Middleware): req - wsgi.Request object """ ctxt = context.get_admin_context() - user = self.auth.get_user_from_access_key(key) + + try: + user = self.auth.get_user_from_access_key(key) + except exception.NotFound: + user = None if user and user.name == username: token_hash = hashlib.sha1('%s%s%f' % (username, key, diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index e50d11a3d..7cb974bb2 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -321,7 +321,10 @@ class FakeAuthManager(object): (user.id == p.project_manager_id)] def get_user_from_access_key(self, key): - return FakeAuthManager.auth_data.get(key, None) + try: + return FakeAuthManager.auth_data[key] + except KeyError: + raise exc.NotFound class FakeRateLimiter(object): diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index aaaa4e415..d1b3c0e10 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -51,11 +51,12 @@ class Test(test.TestCase): def test_authorize_user(self): f = fakes.FakeAuthManager() - f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) + f.add_user('user1_key', + nova.auth.manager.User(1, 'user1', None, None, None)) req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-User'] = 'herp' - req.headers['X-Auth-Key'] = 'derp' + req.headers['X-Auth-User'] = 'user1' + req.headers['X-Auth-Key'] = 'user1_key' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) @@ -65,13 +66,13 @@ class Test(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - u = nova.auth.manager.User(1, 'herp', None, None, None) - f.add_user('derp', u) - f.create_project('test', u) + u = nova.auth.manager.User(1, 'user1', None, None, None) + f.add_user('user1_key', u) + f.create_project('user1_project', u) req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) - req.headers['X-Auth-User'] = 'herp' - req.headers['X-Auth-Key'] = 'derp' + req.headers['X-Auth-User'] = 'user1' + req.headers['X-Auth-Key'] = 'user1_key' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) @@ -92,7 +93,7 @@ class Test(test.TestCase): def test_token_expiry(self): self.destroy_called = False - token_hash = 'bacon' + token_hash = 'token_hash' def destroy_token_mock(meh, context, token): self.destroy_called = True @@ -109,15 +110,26 @@ class Test(test.TestCase): bad_token) req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-Token'] = 'bacon' + req.headers['X-Auth-Token'] = 'token_hash' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') self.assertEqual(self.destroy_called, True) - def test_bad_user(self): + def test_bad_user_bad_key(self): + req = webob.Request.blank('/v1.0/') + req.headers['X-Auth-User'] = 'unknown_user' + req.headers['X-Auth-Key'] = 'unknown_user_key' + result = req.get_response(fakes.wsgi_app()) + self.assertEqual(result.status, '401 Unauthorized') + + def test_bad_user_good_key(self): + f = fakes.FakeAuthManager() + u = nova.auth.manager.User(1, 'user1', None, None, None) + f.add_user('user1_key', u) + req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-User'] = 'herp' - req.headers['X-Auth-Key'] = 'derp' + req.headers['X-Auth-User'] = 'unknown_user' + req.headers['X-Auth-Key'] = 'user1_key' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') @@ -128,7 +140,7 @@ class Test(test.TestCase): def test_bad_token(self): req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-Token'] = 'baconbaconbacon' + req.headers['X-Auth-Token'] = 'unknown_token' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') @@ -137,11 +149,11 @@ class TestFunctional(test.TestCase): def test_token_expiry(self): ctx = context.get_admin_context() tok = db.auth_token_create(ctx, dict( - token_hash='bacon', + token_hash='test_token_hash', cdn_management_url='', server_management_url='', storage_url='', - user_id='ham', + user_id='user1', )) db.auth_token_update(ctx, tok.token_hash, dict( @@ -149,13 +161,13 @@ class TestFunctional(test.TestCase): )) req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-Token'] = 'bacon' + req.headers['X-Auth-Token'] = 'test_token_hash' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') def test_token_doesnotexist(self): req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-Token'] = 'ham' + req.headers['X-Auth-Token'] = 'nonexistant_token_hash' result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') @@ -178,13 +190,13 @@ class TestLimiter(test.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() - u = nova.auth.manager.User(1, 'herp', None, None, None) - f.add_user('derp', u) + u = nova.auth.manager.User(1, 'user1', None, None, None) + f.add_user('user1_key', u) f.create_project('test', u) req = webob.Request.blank('/v1.0/') - req.headers['X-Auth-User'] = 'herp' - req.headers['X-Auth-Key'] = 'derp' + req.headers['X-Auth-User'] = 'user1' + req.headers['X-Auth-Key'] = 'user1_key' result = req.get_response(fakes.wsgi_app()) self.assertEqual(len(result.headers['X-Auth-Token']), 40) -- cgit From 909b72faa77ba0a2bc787309b95fdfae9bb9ca01 Mon Sep 17 00:00:00 2001 From: Brian Lamar Date: Fri, 11 Mar 2011 17:41:10 -0500 Subject: Removed EOL whitespace in accordance with PEP-8. --- nova/tests/api/openstack/test_auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index d1b3c0e10..0448ed701 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -51,7 +51,7 @@ class Test(test.TestCase): def test_authorize_user(self): f = fakes.FakeAuthManager() - f.add_user('user1_key', + f.add_user('user1_key', nova.auth.manager.User(1, 'user1', None, None, None)) req = webob.Request.blank('/v1.0/') -- cgit From 29e59a96602265c5b5746865db94a3f00b8b5cf5 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 11 Mar 2011 15:21:31 -0800 Subject: Change capitalization of Openstack to OpenStack --- nova/tests/integrated/api/client.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/nova/tests/integrated/api/client.py b/nova/tests/integrated/api/client.py index da8d87e07..245eb8c69 100644 --- a/nova/tests/integrated/api/client.py +++ b/nova/tests/integrated/api/client.py @@ -24,7 +24,7 @@ from nova import log as logging LOG = logging.getLogger('nova.tests.api') -class OpenstackApiException(Exception): +class OpenStackApiException(Exception): def __init__(self, message=None, response=None): self.response = response if not message: @@ -37,22 +37,22 @@ class OpenstackApiException(Exception): message = _('%(message)s\nStatus Code: %(_status)s\n' 'Body: %(_body)s') % locals() - super(OpenstackApiException, self).__init__(message) + super(OpenStackApiException, self).__init__(message) -class OpenstackApiAuthenticationException(OpenstackApiException): +class OpenStackApiAuthenticationException(OpenStackApiException): def __init__(self, response=None, message=None): if not message: message = _("Authentication error") - super(OpenstackApiAuthenticationException, self).__init__(message, + super(OpenStackApiAuthenticationException, self).__init__(message, response) -class OpenstackApiNotFoundException(OpenstackApiException): +class OpenStackApiNotFoundException(OpenStackApiException): def __init__(self, response=None, message=None): if not message: message = _("Item not found") - super(OpenstackApiNotFoundException, self).__init__(message, response) + super(OpenStackApiNotFoundException, self).__init__(message, response) class TestOpenStackClient(object): @@ -82,7 +82,7 @@ class TestOpenStackClient(object): conn = httplib.HTTPSConnection(hostname, port=port) else: - raise OpenstackApiException("Unknown scheme: %s" % url) + raise OpenStackApiException("Unknown scheme: %s" % url) relative_url = parsed_url.path if parsed_url.query: @@ -111,7 +111,7 @@ class TestOpenStackClient(object): # Until bug732866 is fixed, we can't check this properly... #if http_status == 401: if http_status != 204: - raise OpenstackApiAuthenticationException(response=response) + raise OpenStackApiAuthenticationException(response=response) auth_headers = {} for k, v in response.getheaders(): @@ -138,9 +138,9 @@ class TestOpenStackClient(object): if check_response_status: if not http_status in check_response_status: if http_status == 404: - raise OpenstackApiNotFoundException(response=response) + raise OpenStackApiNotFoundException(response=response) else: - raise OpenstackApiException( + raise OpenStackApiException( message=_("Unexpected status code"), response=response) -- cgit From 62d7d521273e19d8e700ab301be38830576efa3b Mon Sep 17 00:00:00 2001 From: Ken Pepple Date: Mon, 14 Mar 2011 11:27:21 -0700 Subject: small typo in nova-manage vm live-migration --- bin/nova-manage | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/nova-manage b/bin/nova-manage index 1eb4e5418..2b42dfff5 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -572,7 +572,7 @@ class VmCommands(object): """ ctxt = context.get_admin_context() - instance_id = ec2_id_to_id(ec2_id) + instance_id = ec2utils.ec2_id_to_id(ec2_id) if FLAGS.connection_type != 'libvirt': msg = _('Only KVM is supported for now. Sorry!') -- cgit From f4e7da2d9f7d6793b383c5f187939f19ec849f0a Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 14 Mar 2011 21:10:11 +0100 Subject: Include cpuinfo.xml.template in tarball. --- MANIFEST.in | 1 + 1 file changed, 1 insertion(+) diff --git a/MANIFEST.in b/MANIFEST.in index 2ceed34f3..bf30d1546 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -25,6 +25,7 @@ include nova/db/sqlalchemy/migrate_repo/migrate.cfg include nova/db/sqlalchemy/migrate_repo/README include nova/virt/interfaces.template include nova/virt/libvirt*.xml.template +include nova/virt/cpuinfo.xml.template include nova/tests/CA/ include nova/tests/CA/cacert.pem include nova/tests/CA/private/ -- cgit -- cgit From ce57cff740bc7f821bdbb0dd1367c037b6fa1c01 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 14:02:27 -0700 Subject: The exception is called "ApiError", not "APIError" --- nova/virt/libvirt_conn.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 0b306c950..7994e9547 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -362,19 +362,19 @@ class LibvirtConnection(object): @exception.wrap_exception def pause(self, instance, callback): - raise exception.APIError("pause not supported for libvirt.") + raise exception.ApiError("pause not supported for libvirt.") @exception.wrap_exception def unpause(self, instance, callback): - raise exception.APIError("unpause not supported for libvirt.") + raise exception.ApiError("unpause not supported for libvirt.") @exception.wrap_exception def suspend(self, instance, callback): - raise exception.APIError("suspend not supported for libvirt") + raise exception.ApiError("suspend not supported for libvirt") @exception.wrap_exception def resume(self, instance, callback): - raise exception.APIError("resume not supported for libvirt") + raise exception.ApiError("resume not supported for libvirt") @exception.wrap_exception def rescue(self, instance, callback=None): @@ -779,7 +779,7 @@ class LibvirtConnection(object): 'cpu_time': cpu_time} def get_diagnostics(self, instance_name): - raise exception.APIError(_("diagnostics are not supported " + raise exception.ApiError(_("diagnostics are not supported " "for libvirt")) def get_disks(self, instance_name): -- cgit From 337bda95a9e12d395f838e81e279c875b056aba9 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 14 Mar 2011 22:17:14 +0100 Subject: Add missing fallback chain for ipv6. --- nova/virt/libvirt_conn.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 0b306c950..03f046cbd 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1597,6 +1597,9 @@ class IptablesFirewallDriver(FirewallDriver): self.iptables.ipv4['filter'].add_chain('sg-fallback') self.iptables.ipv4['filter'].add_rule('sg-fallback', '-j DROP') + if FLAGS.use_ipv6: + self.iptables.ipv6['filter'].add_chain('sg-fallback') + self.iptables.ipv6['filter'].add_rule('sg-fallback', '-j DROP') def setup_basic_filtering(self, instance): """Use NWFilter from libvirt for this.""" -- cgit -- cgit From 408a2591d60f5d238e60e4be9197ccc7262f2406 Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Mon, 14 Mar 2011 16:21:33 -0500 Subject: PEP8 cleanup --- nova/tests/db/fakes.py | 4 ++-- nova/tests/test_xenapi.py | 11 +++++------ nova/virt/xenapi/vm_utils.py | 3 +-- 3 files changed, 8 insertions(+), 10 deletions(-) diff --git a/nova/tests/db/fakes.py b/nova/tests/db/fakes.py index 142f6b1c6..5e9a3aa3b 100644 --- a/nova/tests/db/fakes.py +++ b/nova/tests/db/fakes.py @@ -77,8 +77,8 @@ def stub_out_db_instance_api(stubs): 'mac_address': values['mac_address'], 'vcpus': type_data['vcpus'], 'local_gb': type_data['local_gb'], - 'os_type': values['os_type'] - } + 'os_type': values['os_type']} + return FakeModel(base_options) def fake_network_get_by_instance(context, instance_id): diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index cd125a301..8b0affd5c 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -78,8 +78,7 @@ class XenAPIVolumeTestCase(test.TestCase): 'ramdisk_id': 3, 'instance_type': 'm1.large', 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': 'linux' - } + 'os_type': 'linux'} def _create_volume(self, size='0'): """Create a volume object.""" @@ -315,8 +314,8 @@ class XenAPIVMTestCase(test.TestCase): 'ramdisk_id': ramdisk_id, 'instance_type': instance_type, 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': os_type - } + 'os_type': os_type} + conn = xenapi_conn.get_connection(False) instance = db.instance_create(values) conn.spawn(instance) @@ -440,8 +439,8 @@ class XenAPIMigrateInstance(test.TestCase): 'ramdisk_id': None, 'instance_type': 'm1.large', 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': 'linux' - } + 'os_type': 'linux'} + stubs.stub_out_migration_methods(self.stubs) glance_stubs.stubout_glance_client(self.stubs, glance_stubs.FakeGlance) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index f07b57796..763c5fe40 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -136,8 +136,7 @@ class VMHelper(HelperBase): 'VCPUs_at_startup': vcpus, 'VCPUs_max': vcpus, 'VCPUs_params': {}, - 'xenstore_data': {} - } + 'xenstore_data': {}} # Complete VM configuration record according to the image type # non-raw/raw with PV kernel/raw in HVM mode -- cgit From 7c6aa5eb302637dde0d800f7155235858bbabbeb Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 14:32:15 -0700 Subject: Removed duplicated test, renamed same-named (but non-identical) tests --- nova/tests/test_compute.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index e486050be..14559d1dc 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -115,7 +115,7 @@ class ComputeTestCase(test.TestCase): finally: db.instance_destroy(self.context, ref[0]['id']) - def test_create_instance_associates_security_groups(self): + def test_create_instance_associates_security_groups_1(self): """Make sure create associates security groups""" group = self._create_group() instance_ref = models.Instance() @@ -124,18 +124,7 @@ class ComputeTestCase(test.TestCase): instance_ref['hostname'] = 'i-00000001' return instance_ref - def test_create_instance_defaults_display_name(self): - """Verify that an instance cannot be created without a display_name.""" - cases = [dict(), dict(display_name=None)] - for instance in cases: - ref = self.compute_api.create(self.context, - FLAGS.default_instance_type, None, **instance) - try: - self.assertNotEqual(ref[0]['display_name'], None) - finally: - db.instance_destroy(self.context, ref[0]['id']) - - def test_create_instance_associates_security_groups(self): + def test_create_instance_associates_security_groups_2(self): """Make sure create associates security groups""" group = self._create_group() ref = self.compute_api.create( -- cgit From 567e3bc3a7e66896482d83420190a7c4a8df1e5a Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 15:17:00 -0700 Subject: So the first of those tests doesn't pass. Removing as it looks like it was meant to be deleted. --- nova/tests/test_compute.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 14559d1dc..3651f4cef 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -115,16 +115,7 @@ class ComputeTestCase(test.TestCase): finally: db.instance_destroy(self.context, ref[0]['id']) - def test_create_instance_associates_security_groups_1(self): - """Make sure create associates security groups""" - group = self._create_group() - instance_ref = models.Instance() - instance_ref['id'] = 1 - instance_ref['volumes'] = [{'id': 1}, {'id': 2}] - instance_ref['hostname'] = 'i-00000001' - return instance_ref - - def test_create_instance_associates_security_groups_2(self): + def test_create_instance_associates_security_groups(self): """Make sure create associates security groups""" group = self._create_group() ref = self.compute_api.create( -- cgit From a4e94971b696681a5ced189d8f4263c8f77cc531 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 19:57:30 -0700 Subject: Make key_pair optional with OpenStack API --- nova/api/openstack/servers.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index dc28a0782..47ed254ec 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -146,10 +146,14 @@ class Controller(wsgi.Controller): return faults.Fault(exc.HTTPUnprocessableEntity()) context = req.environ['nova.context'] + + key_name = None + key_data = None key_pairs = auth_manager.AuthManager.get_key_pairs(context) - if not key_pairs: - raise exception.NotFound(_("No keypairs defined")) - key_pair = key_pairs[0] + if key_pairs: + key_pair = key_pairs[0] + key_name = key_pair['name'] + key_data = key_pair['public_key'] image_id = common.get_image_id_from_image_hash(self._image_service, context, env['server']['imageId']) @@ -174,8 +178,8 @@ class Controller(wsgi.Controller): ramdisk_id=ramdisk_id, display_name=env['server']['name'], display_description=env['server']['name'], - key_name=key_pair['name'], - key_data=key_pair['public_key'], + key_name=key_name, + key_data=key_data, metadata=metadata, onset_files=env.get('onset_files', [])) -- cgit From da605eb84f7d5de741225ff936447db01690a04f Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 20:48:33 -0700 Subject: Don't generate insecure passwords where it's easy to use urandom instead --- nova/console/manager.py | 2 +- nova/console/xvp.py | 4 ---- nova/utils.py | 15 ++++++++++----- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/nova/console/manager.py b/nova/console/manager.py index 57c75cf4f..bfa571ea9 100644 --- a/nova/console/manager.py +++ b/nova/console/manager.py @@ -69,7 +69,7 @@ class ConsoleProxyManager(manager.Manager): except exception.NotFound: logging.debug(_("Adding console")) if not password: - password = self.driver.generate_password() + password = utils.generate_password(8) if not port: port = self.driver.get_port(context) console_data = {'instance_name': name, diff --git a/nova/console/xvp.py b/nova/console/xvp.py index 68d8c8565..0cedfbb13 100644 --- a/nova/console/xvp.py +++ b/nova/console/xvp.py @@ -91,10 +91,6 @@ class XVPConsoleProxy(object): """Trim password to length, and encode""" return self._xvp_encrypt(password) - def generate_password(self, length=8): - """Returns random console password""" - return os.urandom(length * 2).encode('base64')[:length] - def _rebuild_xvp_conf(self, context): logging.debug(_("Rebuilding xvp conf")) pools = [pool for pool in diff --git a/nova/utils.py b/nova/utils.py index 87e726394..9c8b27d56 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -263,12 +263,17 @@ def generate_mac(): def generate_password(length=20): - """Generate a random sequence of letters and digits - to be used as a password. Note that this is not intended - to represent the ultimate in security. + """Generate a random alphanumeric password, avoiding 'confusing' O,0,I,1. + + Believed to be reasonably secure (with a reasonable password length!) """ - chrs = string.letters + string.digits - return "".join([random.choice(chrs) for i in xrange(length)]) + # 26 letters, 10 digits = 36 + # Remove O, 0, I, 1 => 32 digits + # 32 digits means we're just using the low 5 bit of each byte + chrs = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789" + + random_bytes = os.urandom(length) + return "".join([chrs[ord(random_bytes[i]) % 32] for i in xrange(length)]) def last_octet(address): -- cgit From 3d0cde272e3227978c5875c811c93e1e3df692ed Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 21:01:48 -0700 Subject: Clarify the logic in using 32 symbols --- nova/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 9c8b27d56..0510c3cbe 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -267,9 +267,10 @@ def generate_password(length=20): Believed to be reasonably secure (with a reasonable password length!) """ - # 26 letters, 10 digits = 36 - # Remove O, 0, I, 1 => 32 digits - # 32 digits means we're just using the low 5 bit of each byte + # 26 letters, 10 digits = 36 choices + # Remove O, 0, I, 1 => 32 choices + # 32 choices means we're just using the low 5 bit of each byte, + # so there's no bias introduced by using a modulo chrs = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789" random_bytes = os.urandom(length) -- cgit From 3cf224b9e676b88d1990b13476095be6ec156e5d Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Mon, 14 Mar 2011 21:28:42 -0700 Subject: Fixed problem with metadata creation (backported fix) --- nova/db/sqlalchemy/api.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 56998ce05..d4dd82227 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -746,6 +746,15 @@ def instance_create(context, values): context - request context object values - dict containing column values. """ + metadata = values.get('metadata') + metadata_refs = [] + if metadata: + for metadata_item in metadata: + metadata_ref = models.InstanceMetadata() + metadata_ref.update(metadata_item) + metadata_refs.append(metadata_ref) + values['metadata'] = metadata_refs + instance_ref = models.Instance() instance_ref.update(values) -- cgit From 8a41046dc7cafb19afb6719866b11681daaa9082 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 15 Mar 2011 09:48:21 +0100 Subject: Always put the ipv6 fallback in place. FLAGS.use_ipv6 does not exist yet when the firewall driver is instantiated and the iptables manager takes care not to fiddle with ipv6 if not enabled. --- nova/virt/libvirt_conn.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 03f046cbd..f87decaa0 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -1597,9 +1597,8 @@ class IptablesFirewallDriver(FirewallDriver): self.iptables.ipv4['filter'].add_chain('sg-fallback') self.iptables.ipv4['filter'].add_rule('sg-fallback', '-j DROP') - if FLAGS.use_ipv6: - self.iptables.ipv6['filter'].add_chain('sg-fallback') - self.iptables.ipv6['filter'].add_rule('sg-fallback', '-j DROP') + self.iptables.ipv6['filter'].add_chain('sg-fallback') + self.iptables.ipv6['filter'].add_rule('sg-fallback', '-j DROP') def setup_basic_filtering(self, instance): """Use NWFilter from libvirt for this.""" -- cgit From 56ff68056254610c4f0eb5cd5c5432a68ed30b2f Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 15 Mar 2011 10:42:32 -0700 Subject: Support testing the OpenStack API without key_pairs --- nova/tests/api/openstack/fakes.py | 11 +++++++++-- nova/tests/api/openstack/test_servers.py | 9 ++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index e50d11a3d..ccc853360 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -85,10 +85,17 @@ def wsgi_app(inner_application=None): return mapper -def stub_out_key_pair_funcs(stubs): +def stub_out_key_pair_funcs(stubs, have_key_pair=True): def key_pair(context, user_id): return [dict(name='key', public_key='public_key')] - stubs.Set(nova.db, 'key_pair_get_all_by_user', key_pair) + + def no_key_pair(context, user_id): + return [] + + if have_key_pair: + stubs.Set(nova.db, 'key_pair_get_all_by_user', key_pair) + else: + stubs.Set(nova.db, 'key_pair_get_all_by_user', no_key_pair) def stub_out_image_service(stubs): diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 5d7a208e9..40026a615 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -216,7 +216,7 @@ class ServersTest(test.TestCase): servers = json.loads(res.body)['servers'] self.assertEqual([s['id'] for s in servers], [1, 2]) - def test_create_instance(self): + def _test_create_instance_helper(self, with_key_pair): def instance_create(context, inst): return {'id': '1', 'display_name': 'server_test'} @@ -271,6 +271,13 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) + def test_create_instance(self): + self._test_create_instance_helper(True) + + def test_create_instance_no_key_pair(self): + fakes.stub_out_key_pair_funcs(self.stubs, False) + self._test_create_instance_helper(False) + def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' -- cgit From 22aad6700124411aceed0b2bd3953cbbc48b6130 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Tue, 15 Mar 2011 11:24:07 -0700 Subject: Use random.SystemRandom for easy secure randoms, configurable symbol set by default including mixed-case --- nova/utils.py | 26 ++++++++++++++++---------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 0510c3cbe..199ee8701 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -262,19 +262,25 @@ def generate_mac(): return ':'.join(map(lambda x: "%02x" % x, mac)) -def generate_password(length=20): - """Generate a random alphanumeric password, avoiding 'confusing' O,0,I,1. +# Default symbols to use for passwords. Avoids visually confusing characters. +# ~6 bits per symbol +DEFAULT_PASSWORD_SYMBOLS = ("23456789" # Removed: 0,1 + "ABCDEFGHJKLMNPQRSTUVWXYZ" # Removed: I, O + "abcdefghijkmnopqrstuvwxyz") # Removed: l + + +# ~5 bits per symbol +EASIER_PASSWORD_SYMBOLS = ("23456789" # Removed: 0, 1 + "ABCDEFGHJKLMNPQRSTUVWXYZ") # Removed: I, O + + +def generate_password(length=20, symbols=DEFAULT_PASSWORD_SYMBOLS): + """Generate a random password from the supplied symbols. Believed to be reasonably secure (with a reasonable password length!) """ - # 26 letters, 10 digits = 36 choices - # Remove O, 0, I, 1 => 32 choices - # 32 choices means we're just using the low 5 bit of each byte, - # so there's no bias introduced by using a modulo - chrs = "ABCDEFGHJKLMNPQRSTUVWXYZ23456789" - - random_bytes = os.urandom(length) - return "".join([chrs[ord(random_bytes[i]) % 32] for i in xrange(length)]) + r = random.SystemRandom() + return "".join([r.choice(symbols) for _i in xrange(length)]) def last_octet(address): -- cgit From 1d69d499124317aa1a9cf7d4bc54db2ff0bc3be9 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 15 Mar 2011 14:33:45 -0400 Subject: refactor onset_files quota checking --- nova/compute/api.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/compute/api.py b/nova/compute/api.py index b6ef889f6..c11059a28 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -86,6 +86,8 @@ class API(base.Base): Raises a QuotaError if any limit is exceeded """ + if onset_files is None: + return limit = quota.allowed_onset_files(context) if len(onset_files) > limit: raise quota.QuotaError(code="OnsetFileLimitExceeded") @@ -96,7 +98,6 @@ class API(base.Base): raise quota.QuotaError(code="OnsetFilePathLimitExceeded") if len(content) > content_limit: raise quota.QuotaError(code="OnsetFileContentLimitExceeded") - return onset_files def create(self, context, instance_type, image_id, kernel_id=None, ramdisk_id=None, @@ -142,8 +143,7 @@ class API(base.Base): LOG.warn(msg) raise quota.QuotaError(msg, "MetadataLimitExceeded") - if onset_files is not None: - onset_files = self._check_onset_file_quota(context, onset_files) + self._check_onset_file_quota(context, onset_files) image = self.image_service.show(context, image_id) -- cgit From 0eaf02efd5fef3f77fced9c1a71c32a6f14f293f Mon Sep 17 00:00:00 2001 From: Josh Kearney Date: Tue, 15 Mar 2011 16:21:22 -0500 Subject: Add logging to lock check --- nova/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/utils.py b/nova/utils.py index 87e726394..d6f9ba829 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -518,6 +518,9 @@ def synchronized(name): def wrap(f): @functools.wraps(f) def inner(*args, **kwargs): + LOG.debug(_("Attempting to grab %(lock)s for method " + "%(method)s..." % {"lock": name, + "method": f.__name__})) lock = lockfile.FileLock(os.path.join(FLAGS.lock_path, 'nova-%s.lock' % name)) with lock: -- cgit From e9ef6e04786a40d20f8022bec5d23d2e4503ce3a Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 15 Mar 2011 17:56:00 -0400 Subject: s/onset_files/injected_files/g --- nova/api/openstack/servers.py | 16 ++++----- nova/compute/api.py | 22 ++++++------ nova/compute/manager.py | 2 +- nova/db/sqlalchemy/models.py | 2 +- nova/quota.py | 30 ++++++++-------- nova/tests/api/openstack/test_servers.py | 48 ++++++++++++------------- nova/tests/test_quota.py | 60 ++++++++++++++++---------------- nova/virt/xenapi/vmops.py | 21 +++++------ 8 files changed, 101 insertions(+), 100 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index adb5c5f99..42fe13619 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -169,7 +169,7 @@ class Controller(wsgi.Controller): metadata.append({'key': k, 'value': v}) personality = env['server'].get('personality', []) - onset_files = self._get_onset_files(personality) + injected_files = self._get_injected_files(personality) try: instances = self.compute_api.create( @@ -183,7 +183,7 @@ class Controller(wsgi.Controller): key_name=key_pair['name'], key_data=key_pair['public_key'], metadata=metadata, - onset_files=onset_files) + injected_files=injected_files) except QuotaError as error: self._handle_quota_error(error) @@ -207,15 +207,15 @@ class Controller(wsgi.Controller): else: return self._deserialize(request.body, request.get_content_type()) - def _get_onset_files(self, personality): + def _get_injected_files(self, personality): """ - Create a list of onset files from the personality attribute + Create a list of injected files from the personality attribute - At this time, onset_files must be formatted as a list of + At this time, injected_files must be formatted as a list of (file_path, file_content) pairs for compatibility with the underlying compute service. """ - onset_files = [] + injected_files = [] for item in personality: try: path = item['path'] @@ -230,8 +230,8 @@ class Controller(wsgi.Controller): except TypeError: msg = 'Personality content for %s cannot be decoded' % path raise exc.HTTPBadRequest(explanation=msg) - onset_files.append((path, contents)) - return onset_files + injected_files.append((path, contents)) + return injected_files def _handle_quota_errors(self, error): """ diff --git a/nova/compute/api.py b/nova/compute/api.py index c11059a28..32577af82 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -80,20 +80,20 @@ class API(base.Base): topic, {"method": "get_network_topic", "args": {'fake': 1}}) - def _check_onset_file_quota(self, context, onset_files): + def _check_injected_file_quota(self, context, injected_files): """ - Enforce quota limits on onset files + Enforce quota limits on injected files Raises a QuotaError if any limit is exceeded """ - if onset_files is None: + if injected_files is None: return - limit = quota.allowed_onset_files(context) - if len(onset_files) > limit: + limit = quota.allowed_injected_files(context) + if len(injected_files) > limit: raise quota.QuotaError(code="OnsetFileLimitExceeded") - path_limit = quota.allowed_onset_file_path_bytes(context) - content_limit = quota.allowed_onset_file_content_bytes(context) - for path, content in onset_files: + path_limit = quota.allowed_injected_file_path_bytes(context) + content_limit = quota.allowed_injected_file_content_bytes(context) + for path, content in injected_files: if len(path) > path_limit: raise quota.QuotaError(code="OnsetFilePathLimitExceeded") if len(content) > content_limit: @@ -105,7 +105,7 @@ class API(base.Base): display_name='', display_description='', key_name=None, key_data=None, security_group='default', availability_zone=None, user_data=None, metadata=[], - onset_files=None): + injected_files=None): """Create the number of instances requested if quota and other arguments check out ok.""" @@ -143,7 +143,7 @@ class API(base.Base): LOG.warn(msg) raise quota.QuotaError(msg, "MetadataLimitExceeded") - self._check_onset_file_quota(context, onset_files) + self._check_injected_file_quota(context, injected_files) image = self.image_service.show(context, image_id) @@ -246,7 +246,7 @@ class API(base.Base): "args": {"topic": FLAGS.compute_topic, "instance_id": instance_id, "availability_zone": availability_zone, - "onset_files": onset_files}}) + "injected_files": injected_files}}) for group_id in security_groups: self.trigger_security_group_members_refresh(elevated, group_id) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 6bb169fa5..92deca813 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -179,7 +179,7 @@ class ComputeManager(manager.Manager): """Launch a new instance with specified options.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) - instance_ref.onset_files = kwargs.get('onset_files', []) + instance_ref.injected_files = kwargs.get('injected_files', []) if instance_ref['name'] in self.driver.list_instances(): raise exception.Error(_("Instance has already been created")) LOG.audit(_("instance %s: starting..."), instance_id, diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 162f6fded..1845e85eb 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -161,7 +161,7 @@ class Certificate(BASE, NovaBase): class Instance(BASE, NovaBase): """Represents a guest vm.""" __tablename__ = 'instances' - onset_files = [] + injected_files = [] id = Column(Integer, primary_key=True, autoincrement=True) diff --git a/nova/quota.py b/nova/quota.py index e0fb97542..2b24c0b5b 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -37,12 +37,12 @@ flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') flags.DEFINE_integer('quota_metadata_items', 128, 'number of metadata items allowed per instance') -flags.DEFINE_integer('quota_max_onset_files', 5, - 'number of onset files allowed') -flags.DEFINE_integer('quota_max_onset_file_content_bytes', 10 * 1024, - 'number of bytes allowed per onset file') -flags.DEFINE_integer('quota_max_onset_file_path_bytes', 255, - 'number of bytes allowed per onset file path') +flags.DEFINE_integer('quota_max_injected_files', 5, + 'number of injected files allowed') +flags.DEFINE_integer('quota_max_injected_file_content_bytes', 10 * 1024, + 'number of bytes allowed per injected file') +flags.DEFINE_integer('quota_max_injected_file_path_bytes', 255, + 'number of bytes allowed per injected file path') def get_quota(context, project_id): @@ -113,19 +113,19 @@ def allowed_metadata_items(context, num_metadata_items): return min(num_metadata_items, num_allowed_metadata_items) -def allowed_onset_files(context): - """Return the number of onset files allowed""" - return int(FLAGS.quota_max_onset_files) +def allowed_injected_files(context): + """Return the number of injected files allowed""" + return FLAGS.quota_max_injected_files -def allowed_onset_file_content_bytes(context): - """Return the number of bytes allowed per onset file content""" - return int(FLAGS.quota_max_onset_file_content_bytes) +def allowed_injected_file_content_bytes(context): + """Return the number of bytes allowed per injected file content""" + return FLAGS.quota_max_injected_file_content_bytes -def allowed_onset_file_path_bytes(context): - """Return the number of bytes allowed in an onset file path""" - return int(FLAGS.quota_max_onset_file_path_bytes) +def allowed_injected_file_path_bytes(context): + """Return the number of bytes allowed in an injected file path""" + return FLAGS.quota_max_injected_file_path_bytes class QuotaError(exception.ApiError): diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 7027c7ea3..253b84be9 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -834,13 +834,13 @@ class TestServerInstanceCreation(test.TestCase): class MockComputeAPI(object): def __init__(self): - self.onset_files = None + self.injected_files = None def create(self, *args, **kwargs): - if 'onset_files' in kwargs: - self.onset_files = kwargs['onset_files'] + if 'injected_files' in kwargs: + self.injected_files = kwargs['injected_files'] else: - self.onset_files = None + self.injected_files = None return [{'id': '1234', 'display_name': 'fakeinstance'}] def set_admin_password(self, *args, **kwargs): @@ -920,46 +920,46 @@ class TestServerInstanceCreation(test.TestCase): request = self._get_create_request_json(body_dict) compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) - return request, response, compute_api.onset_files + return request, response, compute_api.injected_files def _create_instance_with_personality_xml(self, personality): body_dict = self._create_personality_request_dict(personality) request = self._get_create_request_xml(body_dict) compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) - return request, response, compute_api.onset_files + return request, response, compute_api.injected_files def test_create_instance_with_no_personality(self): - request, response, onset_files = \ + request, response, injected_files = \ self._create_instance_with_personality_json(personality=None) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, []) + self.assertEquals(injected_files, []) def test_create_instance_with_no_personality_xml(self): - request, response, onset_files = \ + request, response, injected_files = \ self._create_instance_with_personality_xml(personality=None) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, []) + self.assertEquals(injected_files, []) def test_create_instance_with_personality(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Hello, World!"\n' b64contents = base64.b64encode(contents) personality = [(path, b64contents)] - request, response, onset_files = \ + request, response, injected_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [(path, contents)]) + self.assertEquals(injected_files, [(path, contents)]) def test_create_instance_with_personality_xml(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Hello, World!"\n' b64contents = base64.b64encode(contents) personality = [(path, b64contents)] - request, response, onset_files = \ + request, response, injected_files = \ self._create_instance_with_personality_xml(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [(path, contents)]) + self.assertEquals(injected_files, [(path, contents)]) def test_create_instance_with_personality_no_path(self): personality = [('/remove/this/path', @@ -970,7 +970,7 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.onset_files, None) + self.assertEquals(compute_api.injected_files, None) def _test_create_instance_with_personality_no_path_xml(self): personality = [('/remove/this/path', @@ -981,7 +981,7 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.onset_files, None) + self.assertEquals(compute_api.injected_files, None) def test_create_instance_with_personality_no_contents(self): personality = [('/test/path', @@ -992,7 +992,7 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.onset_files, None) + self.assertEquals(compute_api.injected_files, None) def test_create_instance_with_personality_not_a_list(self): personality = [('/test/path', base64.b64encode('test\ncontents\n'))] @@ -1003,16 +1003,16 @@ class TestServerInstanceCreation(test.TestCase): compute_api, response = \ self._run_create_instance_with_mock_compute_api(request) self.assertEquals(response.status_int, 400) - self.assertEquals(compute_api.onset_files, None) + self.assertEquals(compute_api.injected_files, None) def test_create_instance_with_personality_with_non_b64_content(self): path = '/my/file/path' contents = '#!/bin/bash\necho "Oh no!"\n' personality = [(path, contents)] - request, response, onset_files = \ + request, response, injected_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 400) - self.assertEquals(onset_files, None) + self.assertEquals(injected_files, None) def test_create_instance_with_three_personalities(self): files = [ @@ -1023,19 +1023,19 @@ class TestServerInstanceCreation(test.TestCase): personality = [] for path, content in files: personality.append((path, base64.b64encode(content))) - request, response, onset_files = \ + request, response, injected_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, files) + self.assertEquals(injected_files, files) def test_create_instance_personality_empty_content(self): path = '/my/file/path' contents = '' personality = [(path, contents)] - request, response, onset_files = \ + request, response, injected_files = \ self._create_instance_with_personality_json(personality) self.assertEquals(response.status_int, 200) - self.assertEquals(onset_files, [(path, contents)]) + self.assertEquals(injected_files, [(path, contents)]) def test_create_instance_admin_pass_json(self): request, response, dummy = \ diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index d94381aa2..c65bc459d 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -200,66 +200,66 @@ class QuotaTestCase(test.TestCase): image_id='fake', metadata=metadata) - def test_allowed_onset_files(self): + def test_allowed_injected_files(self): self.assertEqual( - quota.allowed_onset_files(self.context), - FLAGS.quota_max_onset_files) + quota.allowed_injected_files(self.context), + FLAGS.quota_max_injected_files) - def _create_with_onset_files(self, files): + def _create_with_injected_files(self, files): api = compute.API(image_service=self.StubImageService()) api.create(self.context, min_count=1, max_count=1, instance_type='m1.small', image_id='fake', - onset_files=files) + injected_files=files) - def test_no_onset_files(self): + def test_no_injected_files(self): api = compute.API(image_service=self.StubImageService()) api.create(self.context, instance_type='m1.small', image_id='fake') - def test_max_onset_files(self): + def test_max_injected_files(self): files = [] - for i in xrange(FLAGS.quota_max_onset_files): + for i in xrange(FLAGS.quota_max_injected_files): files.append(('/my/path%d' % i, 'config = test\n')) - self._create_with_onset_files(files) # no QuotaError + self._create_with_injected_files(files) # no QuotaError - def test_too_many_onset_files(self): + def test_too_many_injected_files(self): files = [] - for i in xrange(FLAGS.quota_max_onset_files + 1): + for i in xrange(FLAGS.quota_max_injected_files + 1): files.append(('/my/path%d' % i, 'my\ncontent%d\n' % i)) self.assertRaises(quota.QuotaError, - self._create_with_onset_files, files) + self._create_with_injected_files, files) - def test_allowed_onset_file_content_bytes(self): + def test_allowed_injected_file_content_bytes(self): self.assertEqual( - quota.allowed_onset_file_content_bytes(self.context), - FLAGS.quota_max_onset_file_content_bytes) + quota.allowed_injected_file_content_bytes(self.context), + FLAGS.quota_max_injected_file_content_bytes) - def test_max_onset_file_content_bytes(self): - max = FLAGS.quota_max_onset_file_content_bytes + def test_max_injected_file_content_bytes(self): + max = FLAGS.quota_max_injected_file_content_bytes content = ''.join(['a' for i in xrange(max)]) files = [('/test/path', content)] - self._create_with_onset_files(files) # no QuotaError + self._create_with_injected_files(files) # no QuotaError - def test_too_many_onset_file_content_bytes(self): - max = FLAGS.quota_max_onset_file_content_bytes + def test_too_many_injected_file_content_bytes(self): + max = FLAGS.quota_max_injected_file_content_bytes content = ''.join(['a' for i in xrange(max + 1)]) files = [('/test/path', content)] self.assertRaises(quota.QuotaError, - self._create_with_onset_files, files) + self._create_with_injected_files, files) - def test_allowed_onset_file_path_bytes(self): + def test_allowed_injected_file_path_bytes(self): self.assertEqual( - quota.allowed_onset_file_path_bytes(self.context), - FLAGS.quota_max_onset_file_path_bytes) + quota.allowed_injected_file_path_bytes(self.context), + FLAGS.quota_max_injected_file_path_bytes) - def test_max_onset_file_path_bytes(self): - max = FLAGS.quota_max_onset_file_path_bytes + def test_max_injected_file_path_bytes(self): + max = FLAGS.quota_max_injected_file_path_bytes path = ''.join(['a' for i in xrange(max)]) files = [(path, 'config = quotatest')] - self._create_with_onset_files(files) # no QuotaError + self._create_with_injected_files(files) # no QuotaError - def test_too_many_onset_file_path_bytes(self): - max = FLAGS.quota_max_onset_file_path_bytes + def test_too_many_injected_file_path_bytes(self): + max = FLAGS.quota_max_injected_file_path_bytes path = ''.join(['a' for i in xrange(max + 1)]) files = [(path, 'config = quotatest')] self.assertRaises(quota.QuotaError, - self._create_with_onset_files, files) + self._create_with_injected_files, files) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index d3fc335fe..488a61e8e 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -137,19 +137,20 @@ class VMOps(object): LOG.info(_('Spawning VM %(instance_name)s created %(vm_ref)s.') % locals()) - def _inject_onset_files(): - onset_files = instance.onset_files - if onset_files: + def _inject_files(): + injected_files = instance.injected_files + if injected_files: # Check if this is a JSON-encoded string and convert if needed. - if isinstance(onset_files, basestring): + if isinstance(injected_files, basestring): try: - onset_files = json.loads(onset_files) + injected_files = json.loads(injected_files) except ValueError: - LOG.exception(_("Invalid value for onset_files: '%s'") - % onset_files) - onset_files = [] + LOG.exception( + _("Invalid value for injected_files: '%s'") + % injected_files) + injected_files = [] # Inject any files, if specified - for path, contents in instance.onset_files: + for path, contents in instance.injected_files: LOG.debug(_("Injecting file path: '%s'") % path) self.inject_file(instance, path, contents) # NOTE(armando): Do we really need to do this in virt? @@ -165,7 +166,7 @@ class VMOps(object): if state == power_state.RUNNING: LOG.debug(_('Instance %s: booted'), instance_name) timer.stop() - _inject_onset_files() + _inject_files() return True except Exception, exc: LOG.warn(exc) -- cgit From 3cc78174e023b3f848b9c4b30468d356ee575ea6 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 15 Mar 2011 18:11:54 -0400 Subject: internationalization --- nova/api/openstack/servers.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 42fe13619..f618c31a0 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -221,15 +221,16 @@ class Controller(wsgi.Controller): path = item['path'] contents = item['contents'] except KeyError as key: - expl = 'Bad personality format: missing %s' % key + expl = _('Bad personality format: missing %s') % key raise exc.HTTPBadRequest(explanation=expl) except TypeError: - raise exc.HTTPBadRequest(explanation='Bad personality format') + expl = _('Bad personality format') + raise exc.HTTPBadRequest(explanation=expl) try: contents = base64.b64decode(contents) except TypeError: - msg = 'Personality content for %s cannot be decoded' % path - raise exc.HTTPBadRequest(explanation=msg) + expl = _('Personality content for %s cannot be decoded') % path + raise exc.HTTPBadRequest(explanation=expl) injected_files.append((path, contents)) return injected_files @@ -238,13 +239,13 @@ class Controller(wsgi.Controller): Reraise quota errors as api-specific http exceptions """ if error.code == "OnsetFileLimitExceeded": - expl = "Personality file limit exceeded" + expl = _("Personality file limit exceeded") raise exc.HTTPBadRequest(explanation=expl) if error.code == "OnsetFilePathLimitExceeded": - expl = "Personality file path too long" + expl = _("Personality file path too long") raise exc.HTTPBadRequest(explanation=expl) if error.code == "OnsetFileContentLimitExceeded": - expl = "Personality file content too long" + expl = _("Personality file content too long") raise exc.HTTPBadRequest(explanation=expl) # if the original error is okay, just reraise it raise error -- cgit From 70769dbe239c979d97154b88a33cb34d377d1196 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 15 Mar 2011 18:12:46 -0400 Subject: pep8 --- nova/tests/api/openstack/test_servers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 253b84be9..a92c0f590 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -806,7 +806,7 @@ ZSBwYXR0ZXJucyBiZWhpbmQgYWxsIGNsb3VkcywgYW5kIHlv\ dSB3aWxsIGtub3csIHRvbywgd2hlbiB5b3UgbGlmdCB5b3Vy\ c2VsZiBoaWdoIGVub3VnaCB0byBzZWUgYmV5b25kIGhvcml6\ b25zLiINCg0KLVJpY2hhcmQgQmFjaA==""", - } + }, ], }} request = self.deserializer.deserialize(serial_request) -- cgit From e237b4a5653384688b16f7fd2c0708eaec4b9ec7 Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 15 Mar 2011 19:11:21 -0400 Subject: ignore differently-named nodes in personality and metadata parsing --- nova/api/openstack/servers.py | 10 +- nova/tests/api/openstack/test_servers.py | 164 ++++++++++++++++++++++--------- 2 files changed, 126 insertions(+), 48 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index f618c31a0..ea88f1fdc 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -575,7 +575,7 @@ class ServerCreateRequestXMLDeserializer(object): if metadata_node is None: return None metadata = {} - for meta_node in metadata_node.childNodes: + for meta_node in self._find_children_named(metadata_node, "meta"): key = meta_node.getAttribute("key") metadata[key] = self._extract_text(meta_node) return metadata @@ -587,7 +587,7 @@ class ServerCreateRequestXMLDeserializer(object): if personality_node is None: return None personality = [] - for file_node in personality_node.childNodes: + for file_node in self._find_children_named(personality_node, "file"): item = {} if file_node.hasAttribute("path"): item["path"] = file_node.getAttribute("path") @@ -602,6 +602,12 @@ class ServerCreateRequestXMLDeserializer(object): return node return None + def _find_children_named(self, parent, name): + """Return all of a nodes children who have the given name""" + for node in parent.childNodes: + if node.nodeName == name: + yield node + def _extract_text(self, node): """Get the text field contained by the given node""" if len(node.childNodes) == 1: diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index a92c0f590..ed37cb705 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -610,7 +610,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_minimal_request(self): serial_request = """ -""" request = self.deserializer.deserialize(serial_request) expected = {"server": { @@ -622,8 +622,10 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_empty_metadata(self): serial_request = """ -""" + + +""" request = self.deserializer.deserialize(serial_request) expected = {"server": { "name": "new-server-test", @@ -635,8 +637,10 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_empty_personality(self): serial_request = """ -""" + + +""" request = self.deserializer.deserialize(serial_request) expected = {"server": { "name": "new-server-test", @@ -648,9 +652,11 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_empty_metadata_and_personality(self): serial_request = """ -\ -""" + + + +""" request = self.deserializer.deserialize(serial_request) expected = {"server": { "name": "new-server-test", @@ -663,9 +669,11 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_empty_metadata_and_personality_reversed(self): serial_request = """ -\ -""" + + + +""" request = self.deserializer.deserialize(serial_request) expected = {"server": { "name": "new-server-test", @@ -678,28 +686,47 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_one_personality(self): serial_request = """ -\ -aabbccdd""" + + + aabbccdd + +""" request = self.deserializer.deserialize(serial_request) expected = [{"path": "/etc/conf", "contents": "aabbccdd"}] self.assertEquals(request["server"]["personality"], expected) def test_request_with_two_personalities(self): serial_request = """ -\ -aabbccdd\ + +aabbccdd abcd""" request = self.deserializer.deserialize(serial_request) expected = [{"path": "/etc/conf", "contents": "aabbccdd"}, {"path": "/etc/sudoers", "contents": "abcd"}] self.assertEquals(request["server"]["personality"], expected) + def test_request_second_personality_node_ignored(self): + serial_request = """ + + + aabbccdd + + + anything + +""" + request = self.deserializer.deserialize(serial_request) + expected = [{"path": "/etc/conf", "contents": "aabbccdd"}] + self.assertEquals(request["server"]["personality"], expected) + + def test_request_with_one_personality_missing_path(self): serial_request = """ -\ + aabbccdd""" request = self.deserializer.deserialize(serial_request) expected = [{"contents": "aabbccdd"}] @@ -707,8 +734,8 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_one_personality_empty_contents(self): serial_request = """ -\ + """ request = self.deserializer.deserialize(serial_request) expected = [{"path": "/etc/conf", "contents": ""}] @@ -716,8 +743,8 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_one_personality_empty_contents_variation(self): serial_request = """ -\ + """ request = self.deserializer.deserialize(serial_request) expected = [{"path": "/etc/conf", "contents": ""}] @@ -725,57 +752,101 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def test_request_with_one_metadata(self): serial_request = """ -\ -beta""" + + + beta + +""" request = self.deserializer.deserialize(serial_request) expected = {"alpha": "beta"} self.assertEquals(request["server"]["metadata"], expected) def test_request_with_two_metadata(self): serial_request = """ -\ -betabar\ -""" + + + beta + bar + +""" request = self.deserializer.deserialize(serial_request) expected = {"alpha": "beta", "foo": "bar"} self.assertEquals(request["server"]["metadata"], expected) def test_request_with_metadata_missing_value(self): serial_request = """ -\ -""" + + + + +""" request = self.deserializer.deserialize(serial_request) expected = {"alpha": ""} self.assertEquals(request["server"]["metadata"], expected) + def test_request_with_two_metadata_missing_value(self): + serial_request = """ + + + + + +""" + request = self.deserializer.deserialize(serial_request) + expected = {"alpha": "", "delta": ""} + self.assertEquals(request["server"]["metadata"], expected) + def test_request_with_metadata_missing_key(self): serial_request = """ -\ -beta""" + + + beta + +""" request = self.deserializer.deserialize(serial_request) expected = {"": "beta"} self.assertEquals(request["server"]["metadata"], expected) + def test_request_with_two_metadata_missing_key(self): + serial_request = """ + + + beta + gamma + +""" + request = self.deserializer.deserialize(serial_request) + expected = {"":"gamma"} + self.assertEquals(request["server"]["metadata"], expected) + def test_request_with_metadata_duplicate_key(self): serial_request = """ -\ -barbaz\ -""" + + + bar + baz + +""" request = self.deserializer.deserialize(serial_request) expected = {"foo": "baz"} self.assertEquals(request["server"]["metadata"], expected) def test_canonical_request_from_docs(self): serial_request = """ -\ -Apache1\ -\ + + + Apache1 + + + \ ICAgICAgDQoiQSBjbG91ZCBkb2VzIG5vdCBrbm93IHdoeSBp\ dCBtb3ZlcyBpbiBqdXN0IHN1Y2ggYSBkaXJlY3Rpb24gYW5k\ IGF0IHN1Y2ggYSBzcGVlZC4uLkl0IGZlZWxzIGFuIGltcHVs\ @@ -784,8 +855,9 @@ QnV0IHRoZSBza3kga25vd3MgdGhlIHJlYXNvbnMgYW5kIHRo\ ZSBwYXR0ZXJucyBiZWhpbmQgYWxsIGNsb3VkcywgYW5kIHlv\ dSB3aWxsIGtub3csIHRvbywgd2hlbiB5b3UgbGlmdCB5b3Vy\ c2VsZiBoaWdoIGVub3VnaCB0byBzZWUgYmV5b25kIGhvcml6\ -b25zLiINCg0KLVJpY2hhcmQgQmFjaA==\ -""" +b25zLiINCg0KLVJpY2hhcmQgQmFjaA== + +""" expected = {"server": { "name": "new-server-test", "imageId": "1", -- cgit From fc07caece79e379b6d6f2a3220806af9271e349b Mon Sep 17 00:00:00 2001 From: Mark Washenberger Date: Tue, 15 Mar 2011 19:23:46 -0400 Subject: pep8 --- nova/tests/api/openstack/test_servers.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index ed37cb705..9a6f2c052 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -722,7 +722,6 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): expected = [{"path": "/etc/conf", "contents": "aabbccdd"}] self.assertEquals(request["server"]["personality"], expected) - def test_request_with_one_personality_missing_path(self): serial_request = """ """ request = self.deserializer.deserialize(serial_request) - expected = {"":"gamma"} + expected = {"": "gamma"} self.assertEquals(request["server"]["metadata"], expected) def test_request_with_metadata_duplicate_key(self): -- cgit From 8b3e35b157c688fd38d5aa0eb10ddef33653003d Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Wed, 16 Mar 2011 10:29:04 +0100 Subject: fixed pep8 errors (with version 0.5.0) --- bin/nova-manage | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/bin/nova-manage b/bin/nova-manage index 44b1d9ac6..c38e25d6b 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -739,8 +739,7 @@ class InstanceCommands(object): _('project'), _('user'), _('zone'), - _('index') - ) + _('index')) if host == None: instances = db.instance_get_all(context.get_admin_context()) @@ -762,8 +761,7 @@ class InstanceCommands(object): instance['project_id'], instance['user_id'], instance['availability_zone'], - instance['launch_index'] - ) + instance['launch_index']) class VolumeCommands(object): @@ -1053,8 +1051,7 @@ CATEGORIES = [ ('instance_type', InstanceTypeCommands), ('image', ImageCommands), ('flavor', InstanceTypeCommands), - ('instance', InstanceCommands) -] + ('instance', InstanceCommands)] def lazy_match(name, key_value_tuples): -- cgit From 016669543a1f6d4ffc281637ba98c6b6fe30be82 Mon Sep 17 00:00:00 2001 From: Thierry Carrez Date: Wed, 16 Mar 2011 10:38:48 +0100 Subject: Fix unknown exception error in euca-get-ajax-console --- nova/virt/libvirt_conn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 7994e9547..d7bdc3faa 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -502,7 +502,7 @@ class LibvirtConnection(object): cmd = 'netcat', '0.0.0.0', port, '-w', '1' try: stdout, stderr = utils.execute(*cmd, process_input='') - except ProcessExecutionError: + except exception.ProcessExecutionError: return port raise Exception(_('Unable to find an open port')) -- cgit From ba35831c1f66c424e9495642ba23e9d2742a339e Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Wed, 16 Mar 2011 10:58:02 +0100 Subject: added correct path to cpu information (tested on a system with 1 installed cpu package) --- nova/virt/libvirt_conn.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 7994e9547..9943b742a 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -984,18 +984,18 @@ class LibvirtConnection(object): xml = self._conn.getCapabilities() xml = libxml2.parseDoc(xml) - nodes = xml.xpathEval('//cpu') + nodes = xml.xpathEval('//host/cpu') if len(nodes) != 1: raise exception.Invalid(_("Invalid xml. '' must be 1," "but %d\n") % len(nodes) + xml.serialize()) cpu_info = dict() - cpu_info['arch'] = xml.xpathEval('//cpu/arch')[0].getContent() - cpu_info['model'] = xml.xpathEval('//cpu/model')[0].getContent() - cpu_info['vendor'] = xml.xpathEval('//cpu/vendor')[0].getContent() + cpu_info['arch'] = xml.xpathEval('//host/cpu/arch')[0].getContent() + cpu_info['model'] = xml.xpathEval('//host/cpu/model')[0].getContent() + cpu_info['vendor'] = xml.xpathEval('//host/cpu/vendor')[0].getContent() - topology_node = xml.xpathEval('//cpu/topology')[0].get_properties() + topology_node = xml.xpathEval('//host/cpu/topology')[0].get_properties() topology = dict() while topology_node != None: name = topology_node.get_name() @@ -1009,7 +1009,7 @@ class LibvirtConnection(object): raise exception.Invalid(_("Invalid xml: topology(%(topology)s) " "must have %(ks)s") % locals()) - feature_nodes = xml.xpathEval('//cpu/feature') + feature_nodes = xml.xpathEval('//host/cpu/feature') features = list() for nodes in feature_nodes: features.append(nodes.get_properties().getContent()) -- cgit From 7fa96f6292ff7d63621fe024b1ef45b1a1996121 Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Wed, 16 Mar 2011 14:49:18 -0400 Subject: Re-commit r804 --- nova/tests/api/openstack/test_servers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 40026a615..cde2fc036 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -216,7 +216,7 @@ class ServersTest(test.TestCase): servers = json.loads(res.body)['servers'] self.assertEqual([s['id'] for s in servers], [1, 2]) - def _test_create_instance_helper(self, with_key_pair): + def _test_create_instance_helper(self): def instance_create(context, inst): return {'id': '1', 'display_name': 'server_test'} @@ -272,11 +272,11 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 200) def test_create_instance(self): - self._test_create_instance_helper(True) + self._test_create_instance_helper() def test_create_instance_no_key_pair(self): - fakes.stub_out_key_pair_funcs(self.stubs, False) - self._test_create_instance_helper(False) + fakes.stub_out_key_pair_funcs(self.stubs, have_key_pair=False) + self._test_create_instance_helper() def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') -- cgit From 663c1726d9a96540b8fd729223fcb34d7cf3cdf7 Mon Sep 17 00:00:00 2001 From: "jaypipes@gmail.com" <> Date: Wed, 16 Mar 2011 14:49:25 -0400 Subject: Re-commit r805 --- nova/tests/api/openstack/test_servers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index cde2fc036..ad36fa551 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -217,6 +217,7 @@ class ServersTest(test.TestCase): self.assertEqual([s['id'] for s in servers], [1, 2]) def _test_create_instance_helper(self): + """Shared implementation for tests below that create instance""" def instance_create(context, inst): return {'id': '1', 'display_name': 'server_test'} -- cgit From 157ea09c03148ff4615bae27ca3f276a05620825 Mon Sep 17 00:00:00 2001 From: Christian Berendt Date: Wed, 16 Mar 2011 19:54:15 +0100 Subject: fixed pep8 issue --- nova/virt/libvirt_conn.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 9943b742a..4e17555f4 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -995,7 +995,8 @@ class LibvirtConnection(object): cpu_info['model'] = xml.xpathEval('//host/cpu/model')[0].getContent() cpu_info['vendor'] = xml.xpathEval('//host/cpu/vendor')[0].getContent() - topology_node = xml.xpathEval('//host/cpu/topology')[0].get_properties() + topology_node = xml.xpathEval('//host/cpu/topology')[0]\ + .get_properties() topology = dict() while topology_node != None: name = topology_node.get_name() -- cgit From 85bae497aa803914d329f2872d343a9982dc370e Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 14:35:04 -0500 Subject: Changes --- nova/api/openstack/flavors.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/api/openstack/flavors.py b/nova/api/openstack/flavors.py index f3d040ba3..b9e40371d 100644 --- a/nova/api/openstack/flavors.py +++ b/nova/api/openstack/flavors.py @@ -36,7 +36,7 @@ class Controller(wsgi.Controller): def index(self, req): """Return all flavors in brief.""" - return dict(flavors=[dict(id=flavor['id'], name=flavor['name']) + return dict(flavors=[dict(id=flavor['flavorid'], name=flavor['name']) for flavor in self.detail(req)['flavors']]) def detail(self, req): @@ -48,6 +48,7 @@ class Controller(wsgi.Controller): """Return data about the given flavor id.""" ctxt = req.environ['nova.context'] values = db.instance_type_get_by_flavor_id(ctxt, id) + values.update({'id': values['flavorid']}) return dict(flavor=values) raise faults.Fault(exc.HTTPNotFound()) -- cgit From 227957e31d75b24bb8afa078c8d3f2bc447a8215 Mon Sep 17 00:00:00 2001 From: "matt.dietz@rackspace.com" <> Date: Wed, 16 Mar 2011 19:40:16 +0000 Subject: Unit test update --- nova/tests/api/openstack/test_flavors.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/tests/api/openstack/test_flavors.py b/nova/tests/api/openstack/test_flavors.py index 8280a505f..30326dc50 100644 --- a/nova/tests/api/openstack/test_flavors.py +++ b/nova/tests/api/openstack/test_flavors.py @@ -15,6 +15,7 @@ # License for the specific language governing permissions and limitations # under the License. +import json import stubout import webob @@ -50,3 +51,5 @@ class FlavorsTest(test.TestCase): req = webob.Request.blank('/v1.0/flavors/1') res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) + body = json.loads(res.body) + self.assertEqual(body['flavor']['id'], 1) -- cgit From f17fb9370d4af42267837a36c937f213669b0291 Mon Sep 17 00:00:00 2001 From: Cerberus Date: Wed, 16 Mar 2011 14:43:57 -0500 Subject: Dumb --- nova/api/openstack/flavors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/openstack/flavors.py b/nova/api/openstack/flavors.py index b9e40371d..1c440b3a9 100644 --- a/nova/api/openstack/flavors.py +++ b/nova/api/openstack/flavors.py @@ -48,7 +48,7 @@ class Controller(wsgi.Controller): """Return data about the given flavor id.""" ctxt = req.environ['nova.context'] values = db.instance_type_get_by_flavor_id(ctxt, id) - values.update({'id': values['flavorid']}) + values['id'] = values['flavorid'] return dict(flavor=values) raise faults.Fault(exc.HTTPNotFound()) -- cgit From a766b4111addad804e47b8be3e6dedb5f80a83c4 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Thu, 17 Mar 2011 02:20:18 +0000 Subject: added in network qos support for xenserver. Pull qos settings from flavor, use when creating instance. --- nova/api/openstack/servers.py | 3 ++- nova/tests/db/fakes.py | 30 ++++++++++++++++++---- nova/tests/test_xenapi.py | 8 ++++++ nova/virt/xenapi/vm_utils.py | 9 ++++--- nova/virt/xenapi/vmops.py | 9 +++++-- plugins/xenserver/xenapi/etc/xapi.d/plugins/glance | 2 +- 6 files changed, 49 insertions(+), 12 deletions(-) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 3ecd4fb01..2f26fa873 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -32,6 +32,7 @@ from nova.api.openstack import faults from nova.auth import manager as auth_manager from nova.compute import instance_types from nova.compute import power_state +from nova.quota import QuotaError import nova.api.openstack @@ -189,7 +190,7 @@ class Controller(wsgi.Controller): metadata=metadata, injected_files=injected_files) except QuotaError as error: - self._handle_quota_error(error) + self._handle_quota_errors(error) server = _translate_keys(instances[0]) password = "%s%s" % (server['server']['name'][:4], diff --git a/nova/tests/db/fakes.py b/nova/tests/db/fakes.py index 5e9a3aa3b..2d25d5fc5 100644 --- a/nova/tests/db/fakes.py +++ b/nova/tests/db/fakes.py @@ -28,13 +28,33 @@ def stub_out_db_instance_api(stubs): """ Stubs out the db API for creating Instances """ INSTANCE_TYPES = { - 'm1.tiny': dict(memory_mb=512, vcpus=1, local_gb=0, flavorid=1), - 'm1.small': dict(memory_mb=2048, vcpus=1, local_gb=20, flavorid=2), + 'm1.tiny': dict(memory_mb=512, + vcpus=1, + local_gb=0, + flavorid=1, + rxtx_cap=1), + 'm1.small': dict(memory_mb=2048, + vcpus=1, + local_gb=20, + flavorid=2, + rxtx_cap=2), 'm1.medium': - dict(memory_mb=4096, vcpus=2, local_gb=40, flavorid=3), - 'm1.large': dict(memory_mb=8192, vcpus=4, local_gb=80, flavorid=4), + dict(memory_mb=4096, + vcpus=2, + local_gb=40, + flavorid=3, + rxtx_cap=3), + 'm1.large': dict(memory_mb=8192, + vcpus=4, + local_gb=80, + flavorid=4, + rxtx_cap=4), 'm1.xlarge': - dict(memory_mb=16384, vcpus=8, local_gb=160, flavorid=5)} + dict(memory_mb=16384, + vcpus=8, + local_gb=160, + flavorid=5, + rxtx_cap=5)} class FakeModel(object): """ Stubs out for model """ diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 8b0affd5c..66a973a78 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -361,6 +361,14 @@ class XenAPIVMTestCase(test.TestCase): glance_stubs.FakeGlance.IMAGE_RAMDISK) self.check_vm_params_for_linux_with_external_kernel() + def test_spawn_with_network_qos(self): + self._create_instance() + for vif_ref in xenapi_fake.get_all('VIF'): + vif_rec = xenapi_fake.get_record('VIF', vif_ref) + self.assertEquals(vif_rec['qos_algorithm_type'], 'ratelimit') + self.assertEquals(vif_rec['qos_algorithm_params']['kbps'], + str(4 * 1024)) + def tearDown(self): super(XenAPIVMTestCase, self).tearDown() self.manager.delete_project(self.project) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 763c5fe40..e0621f73a 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -233,7 +233,9 @@ class VMHelper(HelperBase): raise StorageError(_('Unable to destroy VBD %s') % vbd_ref) @classmethod - def create_vif(cls, session, vm_ref, network_ref, mac_address, dev="0"): + def create_vif(cls, session, vm_ref, network_ref, mac_address, + dev="0", + rxtx_cap=0): """Create a VIF record. Returns a Deferred that gives the new VIF reference.""" vif_rec = {} @@ -243,8 +245,9 @@ class VMHelper(HelperBase): vif_rec['MAC'] = mac_address vif_rec['MTU'] = '1500' vif_rec['other_config'] = {} - vif_rec['qos_algorithm_type'] = '' - vif_rec['qos_algorithm_params'] = {} + vif_rec['qos_algorithm_type'] = "ratelimit" if rxtx_cap else '' + vif_rec['qos_algorithm_params'] = \ + {"kbps": str(rxtx_cap * 1024)} if rxtx_cap else {} LOG.debug(_('Creating VIF for VM %(vm_ref)s,' ' network %(network_ref)s.') % locals()) vif_ref = session.call_xenapi('VIF.create', vif_rec) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 488a61e8e..29f162ad1 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -744,8 +744,12 @@ class VMOps(object): Creates vifs for an instance """ - vm_ref = self._get_vm_opaque_ref(instance.id) + vm_ref = self._get_vm_opaque_ref(instance['id']) + admin_context = context.get_admin_context() + flavor = db.instance_type_get_by_name(admin_context, + instance.instance_type) logging.debug(_("creating vif(s) for vm: |%s|"), vm_ref) + rxtx_cap = flavor['rxtx_cap'] if networks is None: networks = db.network_get_all_by_instance(admin_context, instance['id']) @@ -766,7 +770,8 @@ class VMOps(object): device = "0" VMHelper.create_vif(self._session, vm_ref, network_ref, - instance.mac_address, device) + instance.mac_address, device, + rxtx_cap=rxtx_cap) def reset_network(self, instance): """ diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index c996f6ef4..db39cb0f4 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -216,7 +216,7 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, os_type): 'x-image-meta-status': 'queued', 'x-image-meta-disk-format': 'vhd', 'x-image-meta-container-format': 'ovf', - 'x-image-meta-property-os-type': os_type + 'x-image-meta-property-os-type': os_type, } for header, value in headers.iteritems(): -- cgit From 41619f49ce72d8e85f013c5a5dd248faa8490555 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 17 Mar 2011 11:44:27 +0100 Subject: Comparisons to None should not use == or !=. Stop converting sets to lists before comparing them. They might be in different order after being list()ified. --- nova/virt/libvirt_conn.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 2559c2b81..0a85da541 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -998,14 +998,14 @@ class LibvirtConnection(object): topology_node = xml.xpathEval('//host/cpu/topology')[0]\ .get_properties() topology = dict() - while topology_node != None: + while topology_node: name = topology_node.get_name() topology[name] = topology_node.getContent() topology_node = topology_node.get_next() keys = ['cores', 'sockets', 'threads'] tkeys = topology.keys() - if list(set(tkeys)) != list(set(keys)): + if set(tkeys) != set(keys): ks = ', '.join(keys) raise exception.Invalid(_("Invalid xml: topology(%(topology)s) " "must have %(ks)s") % locals()) -- cgit From 1d64d0a3d7f25448361ce54e32bba3de68c7afd1 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 17 Mar 2011 16:14:59 +0100 Subject: Remove unconditional raise, probably left over from debugging. --- nova/api/ec2/cloud.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 40a9da0e7..e257e44e7 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -959,7 +959,7 @@ class CloudController(object): raise exception.NotFound(_('Image %s not found') % image_id) internal_id = image['id'] del(image['id']) - raise Exception(image) + image['properties']['is_public'] = (operation_type == 'add') return self.image_service.update(context, internal_id, image) -- cgit From 192d3ad7bb9cf49abbca98b0d8e9ae822b204365 Mon Sep 17 00:00:00 2001 From: Monsyne Dragon Date: Thu, 17 Mar 2011 19:34:45 +0000 Subject: fixed code formatting nit. --- nova/virt/xenapi/vm_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index e0621f73a..7dbca321f 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -234,8 +234,7 @@ class VMHelper(HelperBase): @classmethod def create_vif(cls, session, vm_ref, network_ref, mac_address, - dev="0", - rxtx_cap=0): + dev="0", rxtx_cap=0): """Create a VIF record. Returns a Deferred that gives the new VIF reference.""" vif_rec = {} -- cgit