diff options
| author | Donal Lafferty <donal.lafferty@citrix.com> | 2011-06-27 15:22:15 +0100 |
|---|---|---|
| committer | Donal Lafferty <donal.lafferty@citrix.com> | 2011-06-27 15:22:15 +0100 |
| commit | d7844ecbf5dff67d22beaaff90f0bfa3acb4362a (patch) | |
| tree | 413dd7858fdfbd8e214378e3102273f74be7ea71 | |
| parent | 81894967d8b6920f7cf1b64636a7c4f531e81bde (diff) | |
| parent | 8a8c013cd4513b07e936125a23188e7608f40d58 (diff) | |
Pulled trunk, merged boot from ISO changes.
198 files changed, 11781 insertions, 4239 deletions
@@ -17,6 +17,7 @@ Christian Berendt <berendt@b1-systems.de> Chuck Short <zulcss@ubuntu.com> Cory Wright <corywright@gmail.com> Dan Prince <dan.prince@rackspace.com> +Dave Walker <DaveWalker@ubuntu.com> David Pravec <David.Pravec@danix.org> Dean Troyer <dtroyer@gmail.com> Devin Carlen <devin.carlen@gmail.com> @@ -30,6 +31,8 @@ Gabe Westmaas <gabe.westmaas@rackspace.com> Hisaharu Ishii <ishii.hisaharu@lab.ntt.co.jp> Hisaki Ohara <hisaki.ohara@intel.com> Ilya Alekseyev <ialekseev@griddynamics.com> +Isaku Yamahata <yamahata@valinux.co.jp> +Jason Cannavale <jason.cannavale@rackspace.com> Jason Koelker <jason@koelker.net> Jay Pipes <jaypipes@gmail.com> Jesse Andrews <anotherjesse@gmail.com> @@ -58,6 +61,7 @@ Mark Washenberger <mark.washenberger@rackspace.com> Masanori Itoh <itoumsn@nttdata.co.jp> Matt Dietz <matt.dietz@rackspace.com> Michael Gundlach <michael.gundlach@rackspace.com> +Mike Scherbakov <mihgen@gmail.com> Monsyne Dragon <mdragon@rackspace.com> Monty Taylor <mordred@inaugust.com> MORITA Kazutaka <morita.kazutaka@gmail.com> @@ -83,6 +87,7 @@ Trey Morris <trey.morris@rackspace.com> Tushar Patil <tushar.vitthal.patil@gmail.com> Vasiliy Shlykov <vash@vasiliyshlykov.org> Vishvananda Ishaya <vishvananda@gmail.com> +Vivek Y S <vivek.ys@gmail.com> William Wolf <throughnothing@gmail.com> Yoshiaki Tamura <yoshi@midokura.jp> Youcef Laribi <Youcef.Laribi@eu.citrix.com> diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index f42dfd6b5..5926b97de 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -108,6 +108,13 @@ def main(): interface = os.environ.get('DNSMASQ_INTERFACE', FLAGS.dnsmasq_interface) if int(os.environ.get('TESTING', '0')): from nova.tests import fake_flags + + #if FLAGS.fake_rabbit: + # LOG.debug(_("leasing ip")) + # network_manager = utils.import_object(FLAGS.network_manager) + ## reload(fake_flags) + # from nova.tests import fake_flags + action = argv[1] if action in ['add', 'del', 'old']: mac = argv[2] diff --git a/bin/nova-manage b/bin/nova-manage index e3ed7b9d0..02f20347d 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -53,15 +53,14 @@ CLI interface for nova management. """ -import datetime import gettext import glob import json +import netaddr import os import sys import time -import IPy # If ../nova/__init__.py exists, add ../ to Python search path, so that # it will override what happens to be installed in /usr/(local/)lib/python... @@ -78,6 +77,7 @@ from nova import crypto from nova import db from nova import exception from nova import flags +from nova import image from nova import log as logging from nova import quota from nova import rpc @@ -96,8 +96,8 @@ flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('vlan_start', 'nova.network.manager') flags.DECLARE('vpn_start', 'nova.network.manager') flags.DECLARE('fixed_range_v6', 'nova.network.manager') -flags.DECLARE('images_path', 'nova.image.local') -flags.DECLARE('libvirt_type', 'nova.virt.libvirt_conn') +flags.DECLARE('gateway_v6', 'nova.network.manager') +flags.DECLARE('libvirt_type', 'nova.virt.libvirt.connection') flags.DEFINE_flag(flags.HelpFlag()) flags.DEFINE_flag(flags.HelpshortFlag()) flags.DEFINE_flag(flags.HelpXMLFlag()) @@ -257,6 +257,11 @@ class RoleCommands(object): """adds role to user if project is specified, adds project specific role arguments: user, role [project]""" + if project: + projobj = self.manager.get_project(project) + if not projobj.has_member(user): + print "%s not a member of %s" % (user, project) + return self.manager.add_role(user, role, project) def has(self, user, role, project=None): @@ -417,12 +422,16 @@ class ProjectCommands(object): arguments: project_id [key] [value]""" ctxt = context.get_admin_context() if key: + if value.lower() == 'unlimited': + value = None try: db.quota_update(ctxt, project_id, key, value) except exception.ProjectQuotaNotFound: db.quota_create(ctxt, project_id, key, value) - project_quota = quota.get_quota(ctxt, project_id) + project_quota = quota.get_project_quotas(ctxt, project_id) for key, value in project_quota.iteritems(): + if value is None: + value = 'unlimited' print '%s: %s' % (key, value) def remove(self, project_id, user_id): @@ -509,7 +518,7 @@ class FloatingIpCommands(object): def create(self, host, range): """Creates floating ips for host by range arguments: host ip_range""" - for address in IPy.IP(range): + for address in netaddr.IPNetwork(range): db.floating_ip_create(context.get_admin_context(), {'address': str(address), 'host': host}) @@ -517,7 +526,7 @@ class FloatingIpCommands(object): def delete(self, ip_range): """Deletes floating ips by range arguments: range""" - for address in IPy.IP(ip_range): + for address in netaddr.IPNetwork(ip_range): db.floating_ip_destroy(context.get_admin_context(), str(address)) @@ -532,7 +541,7 @@ class FloatingIpCommands(object): for floating_ip in floating_ips: instance = None if floating_ip['fixed_ip']: - instance = floating_ip['fixed_ip']['instance']['ec2_id'] + instance = floating_ip['fixed_ip']['instance']['hostname'] print "%s\t%s\t%s" % (floating_ip['host'], floating_ip['address'], instance) @@ -541,13 +550,10 @@ class FloatingIpCommands(object): class NetworkCommands(object): """Class for managing networks.""" - def create(self, fixed_range=None, num_networks=None, - network_size=None, vlan_start=None, - vpn_start=None, fixed_range_v6=None, label='public'): - """Creates fixed ips for host by range - arguments: fixed_range=FLAG, [num_networks=FLAG], - [network_size=FLAG], [vlan_start=FLAG], - [vpn_start=FLAG], [fixed_range_v6=FLAG]""" + def create(self, fixed_range=None, num_networks=None, network_size=None, + vlan_start=None, vpn_start=None, fixed_range_v6=None, + gateway_v6=None, label='public'): + """Creates fixed ips for host by range""" if not fixed_range: msg = _('Fixed range in the form of 10.0.0.0/8 is ' 'required to create networks.') @@ -563,6 +569,8 @@ class NetworkCommands(object): vpn_start = FLAGS.vpn_start if not fixed_range_v6: fixed_range_v6 = FLAGS.fixed_range_v6 + if not gateway_v6: + gateway_v6 = FLAGS.gateway_v6 net_manager = utils.import_object(FLAGS.network_manager) try: net_manager.create_networks(context.get_admin_context(), @@ -572,6 +580,7 @@ class NetworkCommands(object): vlan_start=int(vlan_start), vpn_start=int(vpn_start), cidr_v6=fixed_range_v6, + gateway_v6=gateway_v6, label=label) except ValueError, e: print e @@ -685,7 +694,7 @@ class ServiceCommands(object): """Show a list of all running services. Filter by host & service name. args: [host] [service]""" ctxt = context.get_admin_context() - now = datetime.datetime.utcnow() + now = utils.utcnow() services = db.service_get_all(ctxt) if host: services = [s for s in services if s['host'] == host] @@ -869,7 +878,7 @@ class InstanceTypeCommands(object): try: instance_types.create(name, memory, vcpus, local_gb, flavorid, swap, rxtx_quota, rxtx_cap) - except exception.InvalidInputException: + except exception.InvalidInput: print "Must supply valid parameters to create instance_type" print e sys.exit(1) @@ -932,7 +941,7 @@ class ImageCommands(object): """Methods for dealing with a cloud in an odd state""" def __init__(self, *args, **kwargs): - self.image_service = utils.import_object(FLAGS.image_service) + self.image_service = image.get_default_image_service() def _register(self, container_format, disk_format, path, owner, name=None, is_public='T', @@ -1051,16 +1060,6 @@ class ImageCommands(object): machine_images = {} other_images = {} directory = os.path.abspath(directory) - # NOTE(vish): If we're importing from the images path dir, attempt - # to move the files out of the way before importing - # so we aren't writing to the same directory. This - # may fail if the dir was a mointpoint. - if (FLAGS.image_service == 'nova.image.local.LocalImageService' - and directory == os.path.abspath(FLAGS.images_path)): - new_dir = "%s_bak" % directory - os.rename(directory, new_dir) - os.mkdir(directory) - directory = new_dir for fn in glob.glob("%s/*/info.json" % directory): try: image_path = os.path.join(fn.rpartition('/')[0], 'image') @@ -1077,24 +1076,100 @@ class ImageCommands(object): self._convert_images(machine_images) +class AgentBuildCommands(object): + """Class for managing agent builds.""" + + def create(self, os, architecture, version, url, md5hash, + hypervisor='xen'): + """Creates a new agent build. + arguments: os architecture version url md5hash [hypervisor='xen']""" + ctxt = context.get_admin_context() + agent_build = db.agent_build_create(ctxt, + {'hypervisor': hypervisor, + 'os': os, + 'architecture': architecture, + 'version': version, + 'url': url, + 'md5hash': md5hash}) + + def delete(self, os, architecture, hypervisor='xen'): + """Deletes an existing agent build. + arguments: os architecture [hypervisor='xen']""" + ctxt = context.get_admin_context() + agent_build_ref = db.agent_build_get_by_triple(ctxt, + hypervisor, os, architecture) + db.agent_build_destroy(ctxt, agent_build_ref['id']) + + def list(self, hypervisor=None): + """Lists all agent builds. + arguments: <none>""" + fmt = "%-10s %-8s %12s %s" + ctxt = context.get_admin_context() + by_hypervisor = {} + for agent_build in db.agent_build_get_all(ctxt): + buildlist = by_hypervisor.get(agent_build.hypervisor) + if not buildlist: + buildlist = by_hypervisor[agent_build.hypervisor] = [] + + buildlist.append(agent_build) + + for key, buildlist in by_hypervisor.iteritems(): + if hypervisor and key != hypervisor: + continue + + print "Hypervisor: %s" % key + print fmt % ('-' * 10, '-' * 8, '-' * 12, '-' * 32) + for agent_build in buildlist: + print fmt % (agent_build.os, agent_build.architecture, + agent_build.version, agent_build.md5hash) + print ' %s' % agent_build.url + + print + + def modify(self, os, architecture, version, url, md5hash, + hypervisor='xen'): + """Update an existing agent build. + arguments: os architecture version url md5hash [hypervisor='xen'] + """ + ctxt = context.get_admin_context() + agent_build_ref = db.agent_build_get_by_triple(ctxt, + hypervisor, os, architecture) + db.agent_build_update(ctxt, agent_build_ref['id'], + {'version': version, + 'url': url, + 'md5hash': md5hash}) + + +class ConfigCommands(object): + """Class for exposing the flags defined by flag_file(s).""" + + def __init__(self): + pass + + def list(self): + print FLAGS.FlagsIntoString() + + CATEGORIES = [ - ('user', UserCommands), ('account', AccountCommands), - ('project', ProjectCommands), - ('role', RoleCommands), - ('shell', ShellCommands), - ('vpn', VpnCommands), + ('agent', AgentBuildCommands), + ('config', ConfigCommands), + ('db', DbCommands), ('fixed', FixedIpCommands), + ('flavor', InstanceTypeCommands), ('floating', FloatingIpCommands), + ('instance_type', InstanceTypeCommands), + ('image', ImageCommands), ('network', NetworkCommands), - ('vm', VmCommands), + ('project', ProjectCommands), + ('role', RoleCommands), ('service', ServiceCommands), - ('db', DbCommands), + ('shell', ShellCommands), + ('user', UserCommands), + ('version', VersionCommands), + ('vm', VmCommands), ('volume', VolumeCommands), - ('instance_type', InstanceTypeCommands), - ('image', ImageCommands), - ('flavor', InstanceTypeCommands), - ('version', VersionCommands)] + ('vpn', VpnCommands)] def lazy_match(name, key_value_tuples): diff --git a/doc/.autogenerated b/doc/.autogenerated deleted file mode 100644 index 456c8ad1e..000000000 --- a/doc/.autogenerated +++ /dev/null @@ -1,283 +0,0 @@ -source/api/nova..adminclient.rst -source/api/nova..api.direct.rst -source/api/nova..api.ec2.admin.rst -source/api/nova..api.ec2.apirequest.rst -source/api/nova..api.ec2.cloud.rst -source/api/nova..api.ec2.metadatarequesthandler.rst -source/api/nova..api.openstack.auth.rst -source/api/nova..api.openstack.backup_schedules.rst -source/api/nova..api.openstack.common.rst -source/api/nova..api.openstack.consoles.rst -source/api/nova..api.openstack.faults.rst -source/api/nova..api.openstack.flavors.rst -source/api/nova..api.openstack.images.rst -source/api/nova..api.openstack.servers.rst -source/api/nova..api.openstack.shared_ip_groups.rst -source/api/nova..api.openstack.zones.rst -source/api/nova..auth.dbdriver.rst -source/api/nova..auth.fakeldap.rst -source/api/nova..auth.ldapdriver.rst -source/api/nova..auth.manager.rst -source/api/nova..auth.signer.rst -source/api/nova..cloudpipe.pipelib.rst -source/api/nova..compute.api.rst -source/api/nova..compute.instance_types.rst -source/api/nova..compute.manager.rst -source/api/nova..compute.monitor.rst -source/api/nova..compute.power_state.rst -source/api/nova..console.api.rst -source/api/nova..console.fake.rst -source/api/nova..console.manager.rst -source/api/nova..console.xvp.rst -source/api/nova..context.rst -source/api/nova..crypto.rst -source/api/nova..db.api.rst -source/api/nova..db.base.rst -source/api/nova..db.migration.rst -source/api/nova..db.sqlalchemy.api.rst -source/api/nova..db.sqlalchemy.migrate_repo.manage.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.005_add_instance_metadata.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.006_add_provider_data_to_volumes.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.007_add_instance_types.rst -source/api/nova..db.sqlalchemy.migration.rst -source/api/nova..db.sqlalchemy.models.rst -source/api/nova..db.sqlalchemy.session.rst -source/api/nova..exception.rst -source/api/nova..fakememcache.rst -source/api/nova..fakerabbit.rst -source/api/nova..flags.rst -source/api/nova..image.glance.rst -source/api/nova..image.local.rst -source/api/nova..image.s3.rst -source/api/nova..image.service.rst -source/api/nova..log.rst -source/api/nova..manager.rst -source/api/nova..network.api.rst -source/api/nova..network.linux_net.rst -source/api/nova..network.manager.rst -source/api/nova..objectstore.bucket.rst -source/api/nova..objectstore.handler.rst -source/api/nova..objectstore.image.rst -source/api/nova..objectstore.stored.rst -source/api/nova..quota.rst -source/api/nova..rpc.rst -source/api/nova..scheduler.chance.rst -source/api/nova..scheduler.driver.rst -source/api/nova..scheduler.manager.rst -source/api/nova..scheduler.simple.rst -source/api/nova..scheduler.zone.rst -source/api/nova..service.rst -source/api/nova..test.rst -source/api/nova..tests.api.openstack.fakes.rst -source/api/nova..tests.api.openstack.test_adminapi.rst -source/api/nova..tests.api.openstack.test_api.rst -source/api/nova..tests.api.openstack.test_auth.rst -source/api/nova..tests.api.openstack.test_common.rst -source/api/nova..tests.api.openstack.test_faults.rst -source/api/nova..tests.api.openstack.test_flavors.rst -source/api/nova..tests.api.openstack.test_images.rst -source/api/nova..tests.api.openstack.test_ratelimiting.rst -source/api/nova..tests.api.openstack.test_servers.rst -source/api/nova..tests.api.openstack.test_shared_ip_groups.rst -source/api/nova..tests.api.openstack.test_zones.rst -source/api/nova..tests.api.test_wsgi.rst -source/api/nova..tests.db.fakes.rst -source/api/nova..tests.declare_flags.rst -source/api/nova..tests.fake_flags.rst -source/api/nova..tests.glance.stubs.rst -source/api/nova..tests.hyperv_unittest.rst -source/api/nova..tests.objectstore_unittest.rst -source/api/nova..tests.real_flags.rst -source/api/nova..tests.runtime_flags.rst -source/api/nova..tests.test_access.rst -source/api/nova..tests.test_api.rst -source/api/nova..tests.test_auth.rst -source/api/nova..tests.test_cloud.rst -source/api/nova..tests.test_compute.rst -source/api/nova..tests.test_console.rst -source/api/nova..tests.test_direct.rst -source/api/nova..tests.test_flags.rst -source/api/nova..tests.test_instance_types.rst -source/api/nova..tests.test_localization.rst -source/api/nova..tests.test_log.rst -source/api/nova..tests.test_middleware.rst -source/api/nova..tests.test_misc.rst -source/api/nova..tests.test_network.rst -source/api/nova..tests.test_quota.rst -source/api/nova..tests.test_rpc.rst -source/api/nova..tests.test_scheduler.rst -source/api/nova..tests.test_service.rst -source/api/nova..tests.test_test.rst -source/api/nova..tests.test_twistd.rst -source/api/nova..tests.test_utils.rst -source/api/nova..tests.test_virt.rst -source/api/nova..tests.test_volume.rst -source/api/nova..tests.test_xenapi.rst -source/api/nova..tests.xenapi.stubs.rst -source/api/nova..twistd.rst -source/api/nova..utils.rst -source/api/nova..version.rst -source/api/nova..virt.connection.rst -source/api/nova..virt.disk.rst -source/api/nova..virt.fake.rst -source/api/nova..virt.hyperv.rst -source/api/nova..virt.images.rst -source/api/nova..virt.libvirt_conn.rst -source/api/nova..virt.xenapi.fake.rst -source/api/nova..virt.xenapi.network_utils.rst -source/api/nova..virt.xenapi.vm_utils.rst -source/api/nova..virt.xenapi.vmops.rst -source/api/nova..virt.xenapi.volume_utils.rst -source/api/nova..virt.xenapi.volumeops.rst -source/api/nova..virt.xenapi_conn.rst -source/api/nova..volume.api.rst -source/api/nova..volume.driver.rst -source/api/nova..volume.manager.rst -source/api/nova..volume.san.rst -source/api/nova..wsgi.rst -source/api/autoindex.rst -source/api/nova..adminclient.rst -source/api/nova..api.direct.rst -source/api/nova..api.ec2.admin.rst -source/api/nova..api.ec2.apirequest.rst -source/api/nova..api.ec2.cloud.rst -source/api/nova..api.ec2.metadatarequesthandler.rst -source/api/nova..api.openstack.auth.rst -source/api/nova..api.openstack.backup_schedules.rst -source/api/nova..api.openstack.common.rst -source/api/nova..api.openstack.consoles.rst -source/api/nova..api.openstack.faults.rst -source/api/nova..api.openstack.flavors.rst -source/api/nova..api.openstack.images.rst -source/api/nova..api.openstack.servers.rst -source/api/nova..api.openstack.shared_ip_groups.rst -source/api/nova..api.openstack.zones.rst -source/api/nova..auth.dbdriver.rst -source/api/nova..auth.fakeldap.rst -source/api/nova..auth.ldapdriver.rst -source/api/nova..auth.manager.rst -source/api/nova..auth.signer.rst -source/api/nova..cloudpipe.pipelib.rst -source/api/nova..compute.api.rst -source/api/nova..compute.instance_types.rst -source/api/nova..compute.manager.rst -source/api/nova..compute.monitor.rst -source/api/nova..compute.power_state.rst -source/api/nova..console.api.rst -source/api/nova..console.fake.rst -source/api/nova..console.manager.rst -source/api/nova..console.xvp.rst -source/api/nova..context.rst -source/api/nova..crypto.rst -source/api/nova..db.api.rst -source/api/nova..db.base.rst -source/api/nova..db.migration.rst -source/api/nova..db.sqlalchemy.api.rst -source/api/nova..db.sqlalchemy.migrate_repo.manage.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.005_add_instance_metadata.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.006_add_provider_data_to_volumes.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.007_add_instance_types.rst -source/api/nova..db.sqlalchemy.migration.rst -source/api/nova..db.sqlalchemy.models.rst -source/api/nova..db.sqlalchemy.session.rst -source/api/nova..exception.rst -source/api/nova..fakememcache.rst -source/api/nova..fakerabbit.rst -source/api/nova..flags.rst -source/api/nova..image.glance.rst -source/api/nova..image.local.rst -source/api/nova..image.s3.rst -source/api/nova..image.service.rst -source/api/nova..log.rst -source/api/nova..manager.rst -source/api/nova..network.api.rst -source/api/nova..network.linux_net.rst -source/api/nova..network.manager.rst -source/api/nova..objectstore.bucket.rst -source/api/nova..objectstore.handler.rst -source/api/nova..objectstore.image.rst -source/api/nova..objectstore.stored.rst -source/api/nova..quota.rst -source/api/nova..rpc.rst -source/api/nova..scheduler.chance.rst -source/api/nova..scheduler.driver.rst -source/api/nova..scheduler.manager.rst -source/api/nova..scheduler.simple.rst -source/api/nova..scheduler.zone.rst -source/api/nova..service.rst -source/api/nova..test.rst -source/api/nova..tests.api.openstack.fakes.rst -source/api/nova..tests.api.openstack.test_adminapi.rst -source/api/nova..tests.api.openstack.test_api.rst -source/api/nova..tests.api.openstack.test_auth.rst -source/api/nova..tests.api.openstack.test_common.rst -source/api/nova..tests.api.openstack.test_faults.rst -source/api/nova..tests.api.openstack.test_flavors.rst -source/api/nova..tests.api.openstack.test_images.rst -source/api/nova..tests.api.openstack.test_ratelimiting.rst -source/api/nova..tests.api.openstack.test_servers.rst -source/api/nova..tests.api.openstack.test_shared_ip_groups.rst -source/api/nova..tests.api.openstack.test_zones.rst -source/api/nova..tests.api.test_wsgi.rst -source/api/nova..tests.db.fakes.rst -source/api/nova..tests.declare_flags.rst -source/api/nova..tests.fake_flags.rst -source/api/nova..tests.glance.stubs.rst -source/api/nova..tests.hyperv_unittest.rst -source/api/nova..tests.objectstore_unittest.rst -source/api/nova..tests.real_flags.rst -source/api/nova..tests.runtime_flags.rst -source/api/nova..tests.test_access.rst -source/api/nova..tests.test_api.rst -source/api/nova..tests.test_auth.rst -source/api/nova..tests.test_cloud.rst -source/api/nova..tests.test_compute.rst -source/api/nova..tests.test_console.rst -source/api/nova..tests.test_direct.rst -source/api/nova..tests.test_flags.rst -source/api/nova..tests.test_instance_types.rst -source/api/nova..tests.test_localization.rst -source/api/nova..tests.test_log.rst -source/api/nova..tests.test_middleware.rst -source/api/nova..tests.test_misc.rst -source/api/nova..tests.test_network.rst -source/api/nova..tests.test_quota.rst -source/api/nova..tests.test_rpc.rst -source/api/nova..tests.test_scheduler.rst -source/api/nova..tests.test_service.rst -source/api/nova..tests.test_test.rst -source/api/nova..tests.test_twistd.rst -source/api/nova..tests.test_utils.rst -source/api/nova..tests.test_virt.rst -source/api/nova..tests.test_volume.rst -source/api/nova..tests.test_xenapi.rst -source/api/nova..tests.xenapi.stubs.rst -source/api/nova..twistd.rst -source/api/nova..utils.rst -source/api/nova..version.rst -source/api/nova..virt.connection.rst -source/api/nova..virt.disk.rst -source/api/nova..virt.fake.rst -source/api/nova..virt.hyperv.rst -source/api/nova..virt.images.rst -source/api/nova..virt.libvirt_conn.rst -source/api/nova..virt.xenapi.fake.rst -source/api/nova..virt.xenapi.network_utils.rst -source/api/nova..virt.xenapi.vm_utils.rst -source/api/nova..virt.xenapi.vmops.rst -source/api/nova..virt.xenapi.volume_utils.rst -source/api/nova..virt.xenapi.volumeops.rst -source/api/nova..virt.xenapi_conn.rst -source/api/nova..volume.api.rst -source/api/nova..volume.driver.rst -source/api/nova..volume.manager.rst -source/api/nova..volume.san.rst -source/api/nova..wsgi.rst diff --git a/doc/source/devref/distributed_scheduler.rst b/doc/source/devref/distributed_scheduler.rst new file mode 100644 index 000000000..e33fda4d2 --- /dev/null +++ b/doc/source/devref/distributed_scheduler.rst @@ -0,0 +1,188 @@ +.. + Copyright 2011 OpenStack LLC + All Rights Reserved. + + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain + a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + License for the specific language governing permissions and limitations + under the License. + + Source for illustrations in doc/source/image_src/zone_distsched_illustrations.odp + (OpenOffice Impress format) Illustrations are "exported" to png and then scaled + to 400x300 or 640x480 as needed and placed in the doc/source/images directory. + +Distributed Scheduler +===================== + +The Scheduler is akin to a Dating Service. Requests for the creation of new instances come in and the most applicable Compute nodes are selected from a large pool of potential candidates. In a small deployment we may be happy with the currently available Chance Scheduler which randomly selects a Host from the available pool. Or if you need something a little more fancy you may want to use the Availability Zone Scheduler, which selects Compute hosts from a logical partitioning of available hosts (within a single Zone). + + .. image:: /images/dating_service.png + +But for larger deployments a more complex scheduling algorithm is required. Additionally, if you are using Zones in your Nova setup, you'll need a scheduler that understand how to pass instance requests from Zone to Zone. + +This is the purpose of the Distributed Scheduler (DS). The DS utilizes the Capabilities of a Zone and its component services to make informed decisions on where a new instance should be created. When making this decision it consults not only all the Compute nodes in the current Zone, but the Compute nodes in each Child Zone. This continues recursively until the ideal host is found. + +So, how does this all work? + +This document will explain the strategy employed by the `ZoneAwareScheduler` and its derivations. You should read the :doc:`devguide/zones` documentation before reading this. + + .. image:: /images/zone_aware_scheduler.png + +Costs & Weights +--------------- +When deciding where to place an Instance, we compare a Weighted Cost for each Host. The Weighting, currently, is just the sum of each Cost. Costs are nothing more than integers from `0 - max_int`. Costs are computed by looking at the various Capabilities of the Host relative to the specs of the Instance being asked for. Trying to put a plain vanilla instance on a high performance host should have a very high cost. But putting a vanilla instance on a vanilla Host should have a low cost. + +Some Costs are more esoteric. Consider a rule that says we should prefer Hosts that don't already have an instance on it that is owned by the user requesting it (to mitigate against machine failures). Here we have to look at all the other Instances on the host to compute our cost. + +An example of some other costs might include selecting: + * a GPU-based host over a standard CPU + * a host with fast ethernet over a 10mbps line + * a host that can run Windows instances + * a host in the EU vs North America + * etc + +This Weight is computed for each Instance requested. If the customer asked for 1000 instances, the consumed resources on each Host are "virtually" depleted so the Cost can change accordingly. + + .. image:: /images/costs_weights.png + +nova.scheduler.zone_aware_scheduler.ZoneAwareScheduler +------------------------------------------------------ +As we explained in the Zones documentation, each Scheduler has a `ZoneManager` object that collects "Capabilities" about child Zones and each of the services running in the current Zone. The `ZoneAwareScheduler` uses this information to make its decisions. + +Here is how it works: + + 1. The compute nodes are filtered and the nodes remaining are weighed. + 2. Filtering the hosts is a simple matter of ensuring the compute node has ample resources (CPU, RAM, Disk, etc) to fulfil the request. + 3. Weighing of the remaining compute nodes assigns a number based on their suitability for the request. + 4. The same request is sent to each child Zone and step #1 is done there too. The resulting weighted list is returned to the parent. + 5. The parent Zone sorts and aggregates all the weights and a final build plan is constructed. + 6. The build plan is executed upon. Concurrently, instance create requests are sent to each of the selected hosts, be they local or in a child zone. Child Zones may forward the requests to their child Zones as needed. + + .. image:: /images/zone_aware_overview.png + +`ZoneAwareScheduler` by itself is not capable of handling all the provisioning itself. Derived classes are used to select which host filtering and weighing strategy will be used. + +Filtering and Weighing +---------------------- +The filtering (excluding compute nodes incapable of fulfilling the request) and weighing (computing the relative "fitness" of a compute node to fulfill the request) rules used are very subjective operations ... Service Providers will probably have a very different set of filtering and weighing rules than private cloud administrators. The filtering and weighing aspects of the `ZoneAwareScheduler` are flexible and extensible. + + .. image:: /images/filtering.png + +Requesting a new instance +------------------------- +Prior to the `ZoneAwareScheduler`, to request a new instance, a call was made to `nova.compute.api.create()`. The type of instance created depended on the value of the `InstanceType` record being passed in. The `InstanceType` determined the amount of disk, CPU, RAM and network required for the instance. Administrators can add new `InstanceType` records to suit their needs. For more complicated instance requests we need to go beyond the default fields in the `InstanceType` table. + +`nova.compute.api.create()` performed the following actions: + 1. it validated all the fields passed into it. + 2. it created an entry in the `Instance` table for each instance requested + 3. it put one `run_instance` message in the scheduler queue for each instance requested + 4. the schedulers picked off the messages and decided which compute node should handle the request. + 5. the `run_instance` message was forwarded to the compute node for processing and the instance is created. + 6. it returned a list of dicts representing each of the `Instance` records (even if the instance has not been activated yet). At least the `instance_ids` are valid. + + .. image:: /images/nova.compute.api.create.png + +Generally, the standard schedulers (like `ChanceScheduler` and `AvailabilityZoneScheduler`) only operate in the current Zone. They have no concept of child Zones. + +The problem with this approach is each request is scattered amongst each of the schedulers. If we are asking for 1000 instances, each scheduler gets the requests one-at-a-time. There is no possability of optimizing the requests to take into account all 1000 instances as a group. We call this Single-Shot vs. All-at-Once. + +For the `ZoneAwareScheduler` we need to use the All-at-Once approach. We need to consider all the hosts across all the Zones before deciding where they should reside. In order to handle this we have a new method `nova.compute.api.create_all_at_once()`. This method does things a little differently: + 1. it validates all the fields passed into it. + 2. it creates a single `reservation_id` for all of instances created. This is a UUID. + 3. it creates a single `run_instance` request in the scheduler queue + 4. a scheduler picks the message off the queue and works on it. + 5. the scheduler sends off an OS API `POST /zones/select` command to each child Zone. The `BODY` payload of the call contains the `request_spec`. + 6. the child Zones use the `request_spec` to compute a weighted list for each instance requested. No attempt to actually create an instance is done at this point. We're only estimating the suitability of the Zones. + 7. if the child Zone has its own child Zones, the `/zones/select` call will be sent down to them as well. + 8. Finally, when all the estimates have bubbled back to the Zone that initiated the call, all the results are merged, sorted and processed. + 9. Now the instances can be created. The initiating Zone either forwards the `run_instance` message to the local Compute node to do the work, or it issues a `POST /servers` call to the relevant child Zone. The parameters to the child Zone call are the same as what was passed in by the user. + 10. The `reservation_id` is passed back to the caller. Later we explain how the user can check on the status of the command with this `reservation_id`. + + .. image:: /images/nova.compute.api.create_all_at_once.png + +The Catch +--------- +This all seems pretty straightforward but, like most things, there's a catch. Zones are expected to operate in complete isolation from each other. Each Zone has its own AMQP service, database and set of Nova services. But, for security reasons Zones should never leak information about the architectural layout internally. That means Zones cannot leak information about hostnames or service IP addresses outside of its world. + +When `POST /zones/select` is called to estimate which compute node to use, time passes until the `POST /servers` call is issued. If we only passed the weight back from the `select` we would have to re-compute the appropriate compute node for the create command ... and we could end up with a different host. Somehow we need to remember the results of our computations and pass them outside of the Zone. Now, we could store this information in the local database and return a reference to it, but remember that the vast majority of weights are going to be ignored. Storing them in the database would result in a flood of disk access and then we have to clean up all these entries periodically. Recall that there are going to be many many `select` calls issued to child Zones asking for estimates. + +Instead, we take a rather innovative approach to the problem. We encrypt all the child zone internal details and pass them back the to parent Zone. If the parent zone decides to use a child Zone for the instance it simply passes the encrypted data back to the child during the `POST /servers` call as an extra parameter. The child Zone can then decrypt the hint and go directly to the Compute node previously selected. If the estimate isn't used, it is simply discarded by the parent. It's for this reason that it is so important that each Zone defines a unique encryption key via `--build_plan_encryption_key` + +In the case of nested child Zones, each Zone re-encrypts the weighted list results and passes those values to the parent. + +Throughout the `nova.api.openstack.servers`, `nova.api.openstack.zones`, `nova.compute.api.create*` and `nova.scheduler.zone_aware_scheduler` code you'll see references to `blob` and `child_blob`. These are the encrypted hints about which Compute node to use. + +Reservation IDs +--------------- + +NOTE: The features described in this section are related to the up-coming 'merge-4' branch. + +The OpenStack API allows a user to list all the instances they own via the `GET /servers/` command or the details on a particular instance via `GET /servers/###`. This mechanism is usually sufficient since OS API only allows for creating one instance at a time, unlike the EC2 API which allows you to specify a quantity of instances to be created. + +NOTE: currently the `GET /servers` command is not Zone-aware since all operations done in child Zones are done via a single administrative account. Therefore, asking a child Zone to `GET /servers` would return all the active instances ... and that would not be what the user intended. Later, when the Keystone Auth system is integrated with Nova, this functionality will be enabled. + +We could use the OS API 1.1 Extensions mechanism to accept a `num_instances` parameter, but this would result in a different return code. Instead of getting back an `Instance` record, we would be getting back a `reservation_id`. So, instead, we've implemented a new command `POST /zones/boot` command which is nearly identical to `POST /servers` except that it takes a `num_instances` parameter and returns a `reservation_id`. Perhaps in OS API 2.x we can unify these approaches. + +Finally, we need to give the user a way to get information on each of the instances created under this `reservation_id`. Fortunately, this is still possible with the existing `GET /servers` command, so long as we add a new optional `reservation_id` parameter. + +`python-novaclient` will be extended to support both of these changes. + +Host Filter +----------- + +As we mentioned earlier, filtering hosts is a very deployment-specific process. Service Providers may have a different set of criteria for filtering Compute nodes than a University. To faciliate this the `nova.scheduler.host_filter` module supports a variety of filtering strategies as well as an easy means for plugging in your own algorithms. + +The filter used is determined by the `--default_host_filter` flag, which points to a Python Class. By default this flag is set to `nova.scheduler.host_filter.AllHostsFilter` which simply returns all available hosts. But there are others: + + * `nova.scheduler.host_filter.InstanceTypeFilter` provides host filtering based on the memory and disk size specified in the `InstanceType` record passed into `run_instance`. + + * `nova.scheduler.host_filter.JSONFilter` filters hosts based on simple JSON expression grammar. Using a LISP-like JSON structure the caller can request instances based on criteria well beyond what `InstanceType` specifies. See `nova.tests.test_host_filter` for examples. + +To create your own `HostFilter` the user simply has to derive from `nova.scheduler.host_filter.HostFilter` and implement two methods: `instance_type_to_filter` and `filter_hosts`. Since Nova is currently dependent on the `InstanceType` structure, the `instance_type_to_filter` method should take an `InstanceType` and turn it into an internal data structure usable by your filter. This is for backward compatibility with existing OpenStack and EC2 API calls. If you decide to create your own call for creating instances not based on `Flavors` or `InstanceTypes` you can ignore this method. The real work is done in `filter_hosts` which must return a list of host tuples for each appropriate host. The set of all available hosts is in the `ZoneManager` object passed into the call as well as the filter query. The host tuple contains (`<hostname>`, `<additional data>`) where `<additional data>` is whatever you want it to be. + +Cost Scheduler Weighing +----------------------- +Every `ZoneAwareScheduler` derivation must also override the `weigh_hosts` method. This takes the list of filtered hosts (generated by the `filter_hosts` method) and returns a list of weight dicts. The weight dicts must contain two keys: `weight` and `hostname` where `weight` is simply an integer (lower is better) and `hostname` is the name of the host. The list does not need to be sorted, this will be done by the `ZoneAwareScheduler` base class when all the results have been assembled. + +Simple Zone Aware Scheduling +---------------------------- +The easiest way to get started with the `ZoneAwareScheduler` is to use the `nova.scheduler.host_filter.HostFilterScheduler`. This scheduler uses the default Host Filter and the `weight_hosts` method simply returns a weight of 1 for all hosts. But, from this, you can see calls being routed from Zone to Zone and follow the flow of things. + +The `--scheduler_driver` flag is how you specify the scheduler class name. + +Flags +----- + +All this Zone and Distributed Scheduler stuff can seem a little daunting to configure, but it's actually not too bad. Here are some of the main flags you should set in your `nova.conf` file: + +:: + + --allow_admin_api=true + --enable_zone_routing=true + --zone_name=zone1 + --build_plan_encryption_key=c286696d887c9aa0611bbb3e2025a45b + --scheduler_driver=nova.scheduler.host_filter.HostFilterScheduler + --default_host_filter=nova.scheduler.host_filter.AllHostsFilter + +`--allow_admin_api` must be set for OS API to enable the new `/zones/*` commands. +`--enable_zone_routing` must be set for OS API commands such as `create()`, `pause()` and `delete()` to get routed from Zone to Zone when looking for instances. +`--zone_name` is only required in child Zones. The default Zone name is `nova`, but you may want to name your child Zones something useful. Duplicate Zone names are not an issue. +`build_plan_encryption_key` is the SHA-256 key for encrypting/decrypting the Host information when it leaves a Zone. Be sure to change this key for each Zone you create. Do not duplicate keys. +`scheduler_driver` is the real workhorse of the operation. For Distributed Scheduler, you need to specify a class derived from `nova.scheduler.zone_aware_scheduler.ZoneAwareScheduler`. +`default_host_filter` is the host filter to be used for filtering candidate Compute nodes. + +Some optional flags which are handy for debugging are: + +:: + + --connection_type=fake + --verbose + +Using the `Fake` virtualization driver is handy when you're setting this stuff up so you're not dealing with a million possible issues at once. When things seem to working correctly, switch back to whatever hypervisor your deployment uses. diff --git a/doc/source/devref/index.rst b/doc/source/devref/index.rst index 9613ba990..0a5a7a4d6 100644 --- a/doc/source/devref/index.rst +++ b/doc/source/devref/index.rst @@ -35,6 +35,7 @@ Programming Concepts .. toctree:: :maxdepth: 3 + zone rabbit API Reference diff --git a/doc/source/devref/zone.rst b/doc/source/devref/zone.rst index 3dd9d37d3..3dc0f80fd 100644 --- a/doc/source/devref/zone.rst +++ b/doc/source/devref/zone.rst @@ -17,11 +17,11 @@ Zones ===== -A Nova deployment is called a Zone. At the very least a Zone requires an API node, a Scheduler node, a database and RabbitMQ. Pushed further a Zone may contain many API nodes, many Scheduler, Volume, Network and Compute nodes as well as a cluster of databases and RabbitMQ servers. A Zone allows you to partition your deployments into logical groups for load balancing and instance distribution. +A Nova deployment is called a Zone. A Zone allows you to partition your deployments into logical groups for load balancing and instance distribution. At the very least a Zone requires an API node, a Scheduler node, a database and RabbitMQ. Pushed further a Zone may contain many API nodes, many Scheduler, Volume, Network and Compute nodes as well as a cluster of databases and RabbitMQ servers. The idea behind Zones is, if a particular deployment is not capable of servicing a particular request, the request may be forwarded to (child) Zones for possible processing. Zones may be nested in a tree fashion. -Zones only know about their immediate children, they do not know about their parent Zones and may in fact have more than one parent. Likewise, a Zone's children may themselves have child Zones. +Zones only know about their immediate children, they do not know about their parent Zones and may in fact have more than one parent. Likewise, a Zone's children may themselves have child Zones and, in those cases, the grandchild's internal structure would not be known to the grand-parent. Zones share nothing. They communicate via the public OpenStack API only. No database, queue, user or project definition is shared between Zones. @@ -34,7 +34,7 @@ Routing between Zones is based on the Capabilities of that Zone. Capabilities ar key=value;value;value, key=value;value;value -Zones have Capabilities which are general to the Zone and are set via `--zone-capabilities` flag. Zones also have dynamic per-service Capabilities. Services derived from `nova.manager.SchedulerDependentManager` (such as Compute, Volume and Network) can set these capabilities by calling the `update_service_capabilities()` method on their `Manager` base class. These capabilities will be periodically sent to the Scheduler service automatically. The rate at which these updates are sent is controlled by the `--periodic_interval` flag. +Zones have Capabilities which are general to the Zone and are set via `--zone_capabilities` flag. Zones also have dynamic per-service Capabilities. Services derived from `nova.manager.SchedulerDependentManager` (such as Compute, Volume and Network) can set these capabilities by calling the `update_service_capabilities()` method on their `Manager` base class. These capabilities will be periodically sent to the Scheduler service automatically. The rate at which these updates are sent is controlled by the `--periodic_interval` flag. Flow within a Zone ------------------ @@ -47,7 +47,7 @@ Inter-service communication within a Zone is done with RabbitMQ. Each class of S These capability messages are received by the Scheduler services and stored in the `ZoneManager` object. The SchedulerManager object has a reference to the `ZoneManager` it can use for load balancing. -The `ZoneManager` also polls the child Zones periodically to gather their capabilities to aid in decision making. This is done via the OpenStack API `/v1.0/zones/info` REST call. This also captures the name of each child Zone. The Zone name is set via the `--zone-name` flag (and defaults to "nova"). +The `ZoneManager` also polls the child Zones periodically to gather their capabilities to aid in decision making. This is done via the OpenStack API `/v1.0/zones/info` REST call. This also captures the name of each child Zone. The Zone name is set via the `--zone_name` flag (and defaults to "nova"). Zone administrative functions ----------------------------- @@ -99,7 +99,7 @@ You can get the `child zone api url`, `nova api key` and `username` from the `no export NOVA_URL="http://192.168.2.120:8774/v1.0/" -This equates to a POST operation to `.../zones/` to add a new zone. No connection attempt to the child zone is done when this command. It only puts an entry in the db at this point. After about 30 seconds the `ZoneManager` in the Scheduler services will attempt to talk to the child zone and get its information. +This equates to a POST operation to `.../zones/` to add a new zone. No connection attempt to the child zone is done with this command. It only puts an entry in the db at this point. After about 30 seconds the `ZoneManager` in the Scheduler services will attempt to talk to the child zone and get its information. Getting a list of child Zones ----------------------------- diff --git a/doc/source/image_src/zones_distsched_illustrations.odp b/doc/source/image_src/zones_distsched_illustrations.odp Binary files differnew file mode 100755 index 000000000..8762a183b --- /dev/null +++ b/doc/source/image_src/zones_distsched_illustrations.odp diff --git a/doc/source/images/costs_weights.png b/doc/source/images/costs_weights.png Binary files differnew file mode 100644 index 000000000..b65e98b0c --- /dev/null +++ b/doc/source/images/costs_weights.png diff --git a/doc/source/images/dating_service.png b/doc/source/images/dating_service.png Binary files differnew file mode 100644 index 000000000..49f1bd86a --- /dev/null +++ b/doc/source/images/dating_service.png diff --git a/doc/source/images/filtering.png b/doc/source/images/filtering.png Binary files differnew file mode 100644 index 000000000..4303bded8 --- /dev/null +++ b/doc/source/images/filtering.png diff --git a/doc/source/images/nova.compute.api.create.png b/doc/source/images/nova.compute.api.create.png Binary files differnew file mode 100755 index 000000000..999f39ed9 --- /dev/null +++ b/doc/source/images/nova.compute.api.create.png diff --git a/doc/source/images/nova.compute.api.create_all_at_once.png b/doc/source/images/nova.compute.api.create_all_at_once.png Binary files differnew file mode 100755 index 000000000..c3ce86d03 --- /dev/null +++ b/doc/source/images/nova.compute.api.create_all_at_once.png diff --git a/doc/source/images/zone_aware_overview.png b/doc/source/images/zone_aware_overview.png Binary files differnew file mode 100755 index 000000000..470e78138 --- /dev/null +++ b/doc/source/images/zone_aware_overview.png diff --git a/doc/source/images/zone_aware_scheduler.png b/doc/source/images/zone_aware_scheduler.png Binary files differnew file mode 100644 index 000000000..a144e1212 --- /dev/null +++ b/doc/source/images/zone_aware_scheduler.png diff --git a/doc/source/man/novamanage.rst b/doc/source/man/novamanage.rst index 9c54f3608..397cc8e80 100644 --- a/doc/source/man/novamanage.rst +++ b/doc/source/man/novamanage.rst @@ -6,7 +6,7 @@ nova-manage control and manage cloud computer instances and images ------------------------------------------------------ -:Author: nova@lists.launchpad.net +:Author: openstack@lists.launchpad.net :Date: 2010-11-16 :Copyright: OpenStack LLC :Version: 0.1 @@ -121,7 +121,7 @@ Nova Role nova-manage role <action> [<argument>] ``nova-manage role add <username> <rolename> <(optional) projectname>`` - Add a user to either a global or project-based role with the indicated <rolename> assigned to the named user. Role names can be one of the following five roles: admin, itsec, projectmanager, netadmin, developer. If you add the project name as the last argument then the role is assigned just for that project, otherwise the user is assigned the named role for all projects. + Add a user to either a global or project-based role with the indicated <rolename> assigned to the named user. Role names can be one of the following five roles: cloudadmin, itsec, sysadmin, netadmin, developer. If you add the project name as the last argument then the role is assigned just for that project, otherwise the user is assigned the named role for all projects. ``nova-manage role has <username> <projectname>`` Checks the user or project and responds with True if the user has a global role with a particular project. diff --git a/doc/source/runnova/managing.users.rst b/doc/source/runnova/managing.users.rst index 392142e86..d3442bed9 100644 --- a/doc/source/runnova/managing.users.rst +++ b/doc/source/runnova/managing.users.rst @@ -38,11 +38,11 @@ Role-based access control (RBAC) is an approach to restricting system access to Nova’s rights management system employs the RBAC model and currently supports the following five roles: -* **Cloud Administrator.** (admin) Users of this class enjoy complete system access. +* **Cloud Administrator.** (cloudadmin) Users of this class enjoy complete system access. * **IT Security.** (itsec) This role is limited to IT security personnel. It permits role holders to quarantine instances. -* **Project Manager.** (projectmanager)The default for project owners, this role affords users the ability to add other users to a project, interact with project images, and launch and terminate instances. +* **System Administrator.** (sysadmin) The default for project owners, this role affords users the ability to add other users to a project, interact with project images, and launch and terminate instances. * **Network Administrator.** (netadmin) Users with this role are permitted to allocate and assign publicly accessible IP addresses as well as create and modify firewall rules. -* **Developer.** This is a general purpose role that is assigned to users by default. +* **Developer.** (developer) This is a general purpose role that is assigned to users by default. RBAC management is exposed through the dashboard for simplified user management. diff --git a/nova/api/direct.py b/nova/api/direct.py index 8ceae299c..ec79151b1 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -42,6 +42,7 @@ from nova import exception from nova import flags from nova import utils from nova import wsgi +import nova.api.openstack.wsgi # Global storage for registering modules. @@ -251,7 +252,7 @@ class Reflection(object): return self._methods[method] -class ServiceWrapper(wsgi.Controller): +class ServiceWrapper(object): """Wrapper to dynamically povide a WSGI controller for arbitrary objects. With lightweight introspection allows public methods on the object to @@ -265,7 +266,7 @@ class ServiceWrapper(wsgi.Controller): def __init__(self, service_handle): self.service_handle = service_handle - @webob.dec.wsgify(RequestClass=wsgi.Request) + @webob.dec.wsgify(RequestClass=nova.api.openstack.wsgi.Request) def __call__(self, req): arg_dict = req.environ['wsgiorg.routing_args'][1] action = arg_dict['action'] @@ -289,8 +290,11 @@ class ServiceWrapper(wsgi.Controller): try: content_type = req.best_match_content_type() - default_xmlns = self.get_default_xmlns(req) - return self._serialize(result, content_type, default_xmlns) + serializer = { + 'application/xml': nova.api.openstack.wsgi.XMLDictSerializer(), + 'application/json': nova.api.openstack.wsgi.JSONDictSerializer(), + }[content_type] + return serializer.serialize(result) except: raise exception.Error("returned non-serializable type: %s" % result) @@ -320,7 +324,7 @@ class Limited(object): def __init__(self, proxy): self._proxy = proxy - if not self.__doc__: + if not self.__doc__: # pylint: disable=E0203 self.__doc__ = proxy.__doc__ if not self._allowed: self._allowed = [] diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index cd59340bd..890d57fe7 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -242,6 +242,7 @@ class Authorizer(wsgi.Middleware): 'CreateKeyPair': ['all'], 'DeleteKeyPair': ['all'], 'DescribeSecurityGroups': ['all'], + 'ImportPublicKey': ['all'], 'AuthorizeSecurityGroupIngress': ['netadmin'], 'RevokeSecurityGroupIngress': ['netadmin'], 'CreateSecurityGroup': ['netadmin'], @@ -327,6 +328,12 @@ class Executor(wsgi.Application): ec2_id = ec2utils.id_to_ec2_id(ex.volume_id, 'vol-%08x') message = _('Volume %s not found') % ec2_id return self._error(req, context, type(ex).__name__, message) + except exception.SnapshotNotFound as ex: + LOG.info(_('SnapshotNotFound raised: %s'), unicode(ex), + context=context) + ec2_id = ec2utils.id_to_ec2_id(ex.snapshot_id, 'snap-%08x') + message = _('Snapshot %s not found') % ec2_id + return self._error(req, context, type(ex).__name__, message) except exception.NotFound as ex: LOG.info(_('NotFound raised: %s'), unicode(ex), context=context) return self._error(req, context, type(ex).__name__, unicode(ex)) @@ -338,6 +345,10 @@ class Executor(wsgi.Application): else: return self._error(req, context, type(ex).__name__, unicode(ex)) + except exception.KeyPairExists as ex: + LOG.debug(_('KeyPairExists raised: %s'), unicode(ex), + context=context) + return self._error(req, context, type(ex).__name__, unicode(ex)) except Exception as ex: extra = {'environment': req.environ} LOG.exception(_('Unexpected error raised: %s'), unicode(ex), diff --git a/nova/api/ec2/admin.py b/nova/api/ec2/admin.py index ea94d9c1f..df7876b9d 100644 --- a/nova/api/ec2/admin.py +++ b/nova/api/ec2/admin.py @@ -22,7 +22,10 @@ Admin API controller, exposed through http via the api worker. import base64 import datetime +import netaddr +import urllib +from nova import compute from nova import db from nova import exception from nova import flags @@ -118,6 +121,9 @@ class AdminController(object): def __str__(self): return 'AdminController' + def __init__(self): + self.compute_api = compute.API() + def describe_instance_types(self, context, **_kwargs): """Returns all active instance types data (vcpus, memory, etc.)""" return {'instanceTypeSet': [instance_dict(v) for v in @@ -305,7 +311,7 @@ class AdminController(object): * Volume Count """ services = db.service_get_all(context, False) - now = datetime.datetime.utcnow() + now = utils.utcnow() hosts = [] rv = [] for host in [service['host'] for service in services]: @@ -326,6 +332,60 @@ class AdminController(object): now)) return {'hosts': rv} - def describe_host(self, _context, name, **_kwargs): - """Returns status info for single node.""" - return host_dict(db.host_get(name)) + def _provider_fw_rule_exists(self, context, rule): + # TODO(todd): we call this repeatedly, can we filter by protocol? + for old_rule in db.provider_fw_rule_get_all(context): + if all([rule[k] == old_rule[k] for k in ('cidr', 'from_port', + 'to_port', 'protocol')]): + return True + return False + + def block_external_addresses(self, context, cidr): + """Add provider-level firewall rules to block incoming traffic.""" + LOG.audit(_('Blocking traffic to all projects incoming from %s'), + cidr, context=context) + cidr = urllib.unquote(cidr).decode() + # raise if invalid + netaddr.IPNetwork(cidr) + rule = {'cidr': cidr} + tcp_rule = rule.copy() + tcp_rule.update({'protocol': 'tcp', 'from_port': 1, 'to_port': 65535}) + udp_rule = rule.copy() + udp_rule.update({'protocol': 'udp', 'from_port': 1, 'to_port': 65535}) + icmp_rule = rule.copy() + icmp_rule.update({'protocol': 'icmp', 'from_port': -1, + 'to_port': None}) + rules_added = 0 + if not self._provider_fw_rule_exists(context, tcp_rule): + db.provider_fw_rule_create(context, tcp_rule) + rules_added += 1 + if not self._provider_fw_rule_exists(context, udp_rule): + db.provider_fw_rule_create(context, udp_rule) + rules_added += 1 + if not self._provider_fw_rule_exists(context, icmp_rule): + db.provider_fw_rule_create(context, icmp_rule) + rules_added += 1 + if not rules_added: + raise exception.ApiError(_('Duplicate rule')) + self.compute_api.trigger_provider_fw_rules_refresh(context) + return {'status': 'OK', 'message': 'Added %s rules' % rules_added} + + def describe_external_address_blocks(self, context): + blocks = db.provider_fw_rule_get_all(context) + # NOTE(todd): use a set since we have icmp/udp/tcp rules with same cidr + blocks = set([b.cidr for b in blocks]) + blocks = [{'cidr': b} for b in blocks] + return {'externalIpBlockInfo': + list(sorted(blocks, key=lambda k: k['cidr']))} + + def remove_external_address_block(self, context, cidr): + LOG.audit(_('Removing ip block from %s'), cidr, context=context) + cidr = urllib.unquote(cidr).decode() + # raise if invalid + netaddr.IPNetwork(cidr) + rules = db.provider_fw_rule_get_all_by_cidr(context, cidr) + for rule in rules: + db.provider_fw_rule_destroy(context, rule['id']) + if rules: + self.compute_api.trigger_provider_fw_rules_refresh(context) + return {'status': 'OK', 'message': 'Deleted %s rules' % len(rules)} diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py index 6672e60bb..7d78c5cfa 100644 --- a/nova/api/ec2/apirequest.py +++ b/nova/api/ec2/apirequest.py @@ -21,22 +21,15 @@ APIRequest class """ import datetime -import re # TODO(termie): replace minidom with etree from xml.dom import minidom from nova import log as logging +from nova.api.ec2 import ec2utils LOG = logging.getLogger("nova.api.request") -_c2u = re.compile('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))') - - -def _camelcase_to_underscore(str): - return _c2u.sub(r'_\1', str).lower().strip('_') - - def _underscore_to_camelcase(str): return ''.join([x[:1].upper() + x[1:] for x in str.split('_')]) @@ -51,59 +44,6 @@ def _database_to_isoformat(datetimeobj): return datetimeobj.strftime("%Y-%m-%dT%H:%M:%SZ") -def _try_convert(value): - """Return a non-string from a string or unicode, if possible. - - ============= ===================================================== - When value is returns - ============= ===================================================== - zero-length '' - 'None' None - 'True' True - 'False' False - '0', '-0' 0 - 0xN, -0xN int from hex (postitive) (N is any number) - 0bN, -0bN int from binary (positive) (N is any number) - * try conversion to int, float, complex, fallback value - - """ - if len(value) == 0: - return '' - if value == 'None': - return None - if value == 'True': - return True - if value == 'False': - return False - valueneg = value[1:] if value[0] == '-' else value - if valueneg == '0': - return 0 - if valueneg == '': - return value - if valueneg[0] == '0': - if valueneg[1] in 'xX': - return int(value, 16) - elif valueneg[1] in 'bB': - return int(value, 2) - else: - try: - return int(value, 8) - except ValueError: - pass - try: - return int(value) - except ValueError: - pass - try: - return float(value) - except ValueError: - pass - try: - return complex(value) - except ValueError: - return value - - class APIRequest(object): def __init__(self, controller, action, version, args): self.controller = controller @@ -114,7 +54,7 @@ class APIRequest(object): def invoke(self, context): try: method = getattr(self.controller, - _camelcase_to_underscore(self.action)) + ec2utils.camelcase_to_underscore(self.action)) except AttributeError: controller = self.controller action = self.action @@ -125,19 +65,7 @@ class APIRequest(object): # and reraise as 400 error. raise Exception(_error) - args = {} - for key, value in self.args.items(): - parts = key.split(".") - key = _camelcase_to_underscore(parts[0]) - if isinstance(value, str) or isinstance(value, unicode): - # NOTE(vish): Automatically convert strings back - # into their respective values - value = _try_convert(value) - if len(parts) > 1: - d = args.get(key, {}) - d[parts[1]] = value - value = d - args[key] = value + args = ec2utils.dict_from_dotted_str(self.args.items()) for key in args.keys(): # NOTE(vish): Turn numeric dict keys into lists diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index c35b6024e..9aaf37a2d 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -23,8 +23,7 @@ datastore. """ import base64 -import datetime -import IPy +import netaddr import os import urllib import tempfile @@ -40,6 +39,7 @@ from nova import flags from nova import ipv6 from nova import log as logging from nova import network +from nova import rpc from nova import utils from nova import volume from nova.api.ec2 import ec2utils @@ -137,6 +137,13 @@ class CloudController(object): return services[0]['availability_zone'] return 'unknown zone' + def _get_image_state(self, image): + # NOTE(vish): fallback status if image_state isn't set + state = image.get('status') + if state == 'active': + state = 'available' + return image['properties'].get('image_state', state) + def get_metadata(self, address): ctxt = context.get_admin_context() instance_ref = self.compute_api.get_all(ctxt, fixed_ip=address) @@ -159,7 +166,7 @@ class CloudController(object): floating_ip = db.instance_get_floating_address(ctxt, instance_ref['id']) ec2_id = ec2utils.id_to_ec2_id(instance_ref['id']) - image_ec2_id = self.image_ec2_id(instance_ref['image_id']) + image_ec2_id = self.image_ec2_id(instance_ref['image_ref']) data = { 'user-data': base64.b64decode(instance_ref['user_data']), 'meta-data': { @@ -235,7 +242,7 @@ class CloudController(object): 'zoneState': 'available'}]} services = db.service_get_all(context, False) - now = datetime.datetime.utcnow() + now = utils.utcnow() hosts = [] for host in [service['host'] for service in services]: if not host in hosts: @@ -283,14 +290,50 @@ class CloudController(object): owner=None, restorable_by=None, **kwargs): - return {'snapshotSet': [{'snapshotId': 'fixme', - 'volumeId': 'fixme', - 'status': 'fixme', - 'startTime': 'fixme', - 'progress': 'fixme', - 'ownerId': 'fixme', - 'volumeSize': 0, - 'description': 'fixme'}]} + if snapshot_id: + snapshots = [] + for ec2_id in snapshot_id: + internal_id = ec2utils.ec2_id_to_id(ec2_id) + snapshot = self.volume_api.get_snapshot( + context, + snapshot_id=internal_id) + snapshots.append(snapshot) + else: + snapshots = self.volume_api.get_all_snapshots(context) + snapshots = [self._format_snapshot(context, s) for s in snapshots] + return {'snapshotSet': snapshots} + + def _format_snapshot(self, context, snapshot): + s = {} + s['snapshotId'] = ec2utils.id_to_ec2_id(snapshot['id'], 'snap-%08x') + s['volumeId'] = ec2utils.id_to_ec2_id(snapshot['volume_id'], + 'vol-%08x') + s['status'] = snapshot['status'] + s['startTime'] = snapshot['created_at'] + s['progress'] = snapshot['progress'] + s['ownerId'] = snapshot['project_id'] + s['volumeSize'] = snapshot['volume_size'] + s['description'] = snapshot['display_description'] + + s['display_name'] = snapshot['display_name'] + s['display_description'] = snapshot['display_description'] + return s + + def create_snapshot(self, context, volume_id, **kwargs): + LOG.audit(_("Create snapshot of volume %s"), volume_id, + context=context) + volume_id = ec2utils.ec2_id_to_id(volume_id) + snapshot = self.volume_api.create_snapshot( + context, + volume_id=volume_id, + name=kwargs.get('display_name'), + description=kwargs.get('display_description')) + return self._format_snapshot(context, snapshot) + + def delete_snapshot(self, context, snapshot_id, **kwargs): + snapshot_id = ec2utils.ec2_id_to_id(snapshot_id) + self.volume_api.delete_snapshot(context, snapshot_id=snapshot_id) + return True def describe_key_pairs(self, context, key_name=None, **kwargs): key_pairs = db.key_pair_get_all_by_user(context, context.user_id) @@ -409,7 +452,7 @@ class CloudController(object): elif cidr_ip: # If this fails, it throws an exception. This is what we want. cidr_ip = urllib.unquote(cidr_ip).decode() - IPy.IP(cidr_ip) + netaddr.IPNetwork(cidr_ip) values['cidr'] = cidr_ip else: values['cidr'] = '0.0.0.0/0' @@ -559,7 +602,7 @@ class CloudController(object): instance_id = ec2utils.ec2_id_to_id(ec2_id) output = self.compute_api.get_console_output( context, instance_id=instance_id) - now = datetime.datetime.utcnow() + now = utils.utcnow() return {"InstanceId": ec2_id, "Timestamp": now, "output": base64.b64encode(output)} @@ -619,16 +662,30 @@ class CloudController(object): 'volumeId': v['volumeId']}] else: v['attachmentSet'] = [{}] + if volume.get('snapshot_id') != None: + v['snapshotId'] = ec2utils.id_to_ec2_id(volume['snapshot_id'], + 'snap-%08x') + else: + v['snapshotId'] = None v['display_name'] = volume['display_name'] v['display_description'] = volume['display_description'] return v - def create_volume(self, context, size, **kwargs): - LOG.audit(_("Create volume of %s GB"), size, context=context) + def create_volume(self, context, **kwargs): + size = kwargs.get('size') + if kwargs.get('snapshot_id') != None: + snapshot_id = ec2utils.ec2_id_to_id(kwargs['snapshot_id']) + LOG.audit(_("Create volume from snapshot %s"), snapshot_id, + context=context) + else: + snapshot_id = None + LOG.audit(_("Create volume of %s GB"), size, context=context) + volume = self.volume_api.create( context, size=size, + snapshot_id=snapshot_id, name=kwargs.get('display_name'), description=kwargs.get('display_description')) # TODO(vish): Instance should be None at db layer instead of @@ -724,13 +781,13 @@ class CloudController(object): instances = self.compute_api.get_all(context, **kwargs) for instance in instances: if not context.is_admin: - if instance['image_id'] == str(FLAGS.vpn_image_id): + if instance['image_ref'] == str(FLAGS.vpn_image_id): continue i = {} instance_id = instance['id'] ec2_id = ec2utils.id_to_ec2_id(instance_id) i['instanceId'] = ec2_id - i['imageId'] = self.image_ec2_id(instance['image_id']) + i['imageId'] = self.image_ec2_id(instance['image_ref']) i['instanceState'] = { 'code': instance['state'], 'name': instance['state_description']} @@ -816,8 +873,14 @@ class CloudController(object): def allocate_address(self, context, **kwargs): LOG.audit(_("Allocate address"), context=context) - public_ip = self.network_api.allocate_floating_ip(context) - return {'publicIp': public_ip} + try: + public_ip = self.network_api.allocate_floating_ip(context) + return {'publicIp': public_ip} + except rpc.RemoteError as ex: + if ex.exc_type == 'NoMoreAddresses': + raise exception.NoMoreFloatingIps() + else: + raise def release_address(self, context, public_ip, **kwargs): LOG.audit(_("Release address %s"), public_ip, context=context) @@ -846,10 +909,39 @@ class CloudController(object): if kwargs.get('ramdisk_id'): ramdisk = self._get_image(context, kwargs['ramdisk_id']) kwargs['ramdisk_id'] = ramdisk['id'] + for bdm in kwargs.get('block_device_mapping', []): + # NOTE(yamahata) + # BlockDevicedMapping.<N>.DeviceName + # BlockDevicedMapping.<N>.Ebs.SnapshotId + # BlockDevicedMapping.<N>.Ebs.VolumeSize + # BlockDevicedMapping.<N>.Ebs.DeleteOnTermination + # BlockDevicedMapping.<N>.VirtualName + # => remove .Ebs and allow volume id in SnapshotId + ebs = bdm.pop('ebs', None) + if ebs: + ec2_id = ebs.pop('snapshot_id') + id = ec2utils.ec2_id_to_id(ec2_id) + if ec2_id.startswith('snap-'): + bdm['snapshot_id'] = id + elif ec2_id.startswith('vol-'): + bdm['volume_id'] = id + ebs.setdefault('delete_on_termination', True) + bdm.update(ebs) + + image = self._get_image(context, kwargs['image_id']) + + if image: + image_state = self._get_image_state(image) + else: + raise exception.ImageNotFound(image_id=kwargs['image_id']) + + if image_state != 'available': + raise exception.ApiError(_('Image must be available')) + instances = self.compute_api.create(context, instance_type=instance_types.get_instance_type_by_name( kwargs.get('instance_type', None)), - image_id=self._get_image(context, kwargs['image_id'])['id'], + image_href=self._get_image(context, kwargs['image_id'])['id'], min_count=int(kwargs.get('min_count', max_count)), max_count=max_count, kernel_id=kwargs.get('kernel_id'), @@ -860,37 +952,54 @@ class CloudController(object): user_data=kwargs.get('user_data'), security_group=kwargs.get('security_group'), availability_zone=kwargs.get('placement', {}).get( - 'AvailabilityZone')) + 'AvailabilityZone'), + block_device_mapping=kwargs.get('block_device_mapping', {})) return self._format_run_instances(context, instances[0]['reservation_id']) + def _do_instance(self, action, context, ec2_id): + instance_id = ec2utils.ec2_id_to_id(ec2_id) + action(context, instance_id=instance_id) + + def _do_instances(self, action, context, instance_id): + for ec2_id in instance_id: + self._do_instance(action, context, ec2_id) + def terminate_instances(self, context, instance_id, **kwargs): """Terminate each instance in instance_id, which is a list of ec2 ids. instance_id is a kwarg so its name cannot be modified.""" LOG.debug(_("Going to start terminating instances")) - for ec2_id in instance_id: - instance_id = ec2utils.ec2_id_to_id(ec2_id) - self.compute_api.delete(context, instance_id=instance_id) + self._do_instances(self.compute_api.delete, context, instance_id) return True def reboot_instances(self, context, instance_id, **kwargs): """instance_id is a list of instance ids""" LOG.audit(_("Reboot instance %r"), instance_id, context=context) - for ec2_id in instance_id: - instance_id = ec2utils.ec2_id_to_id(ec2_id) - self.compute_api.reboot(context, instance_id=instance_id) + self._do_instances(self.compute_api.reboot, context, instance_id) + return True + + def stop_instances(self, context, instance_id, **kwargs): + """Stop each instances in instance_id. + Here instance_id is a list of instance ids""" + LOG.debug(_("Going to stop instances")) + self._do_instances(self.compute_api.stop, context, instance_id) + return True + + def start_instances(self, context, instance_id, **kwargs): + """Start each instances in instance_id. + Here instance_id is a list of instance ids""" + LOG.debug(_("Going to start instances")) + self._do_instances(self.compute_api.start, context, instance_id) return True def rescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" - instance_id = ec2utils.ec2_id_to_id(instance_id) - self.compute_api.rescue(context, instance_id=instance_id) + self._do_instance(self.compute_api.rescue, contect, instnace_id) return True def unrescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" - instance_id = ec2utils.ec2_id_to_id(instance_id) - self.compute_api.unrescue(context, instance_id=instance_id) + self._do_instance(self.compute_api.unrescue, context, instance_id) return True def update_instance(self, context, instance_id, **kwargs): @@ -901,7 +1010,8 @@ class CloudController(object): changes[field] = kwargs[field] if changes: instance_id = ec2utils.ec2_id_to_id(instance_id) - self.compute_api.update(context, instance_id=instance_id, **kwargs) + self.compute_api.update(context, instance_id=instance_id, + **changes) return True @staticmethod @@ -925,7 +1035,12 @@ class CloudController(object): def image_ec2_id(image_id, image_type='ami'): """Returns image ec2_id using id and three letter type.""" template = image_type + '-%08x' - return ec2utils.id_to_ec2_id(int(image_id), template=template) + try: + return ec2utils.id_to_ec2_id(int(image_id), template=template) + except ValueError: + #TODO(wwolf): once we have ec2_id -> glance_id mapping + # in place, this wont be necessary + return "ami-00000000" def _get_image(self, context, ec2_id): try: @@ -956,11 +1071,8 @@ class CloudController(object): get('image_location'), name) else: i['imageLocation'] = image['properties'].get('image_location') - # NOTE(vish): fallback status if image_state isn't set - state = image.get('status') - if state == 'active': - state = 'available' - i['imageState'] = image['properties'].get('image_state', state) + + i['imageState'] = self._get_image_state(image) i['displayName'] = name i['description'] = image.get('description') display_mapping = {'aki': 'kernel', diff --git a/nova/api/ec2/ec2utils.py b/nova/api/ec2/ec2utils.py index 163aa4ed2..222e1de1e 100644 --- a/nova/api/ec2/ec2utils.py +++ b/nova/api/ec2/ec2utils.py @@ -16,6 +16,8 @@ # License for the specific language governing permissions and limitations # under the License. +import re + from nova import exception @@ -30,3 +32,95 @@ def ec2_id_to_id(ec2_id): def id_to_ec2_id(instance_id, template='i-%08x'): """Convert an instance ID (int) to an ec2 ID (i-[base 16 number])""" return template % instance_id + + +_c2u = re.compile('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))') + + +def camelcase_to_underscore(str): + return _c2u.sub(r'_\1', str).lower().strip('_') + + +def _try_convert(value): + """Return a non-string from a string or unicode, if possible. + + ============= ===================================================== + When value is returns + ============= ===================================================== + zero-length '' + 'None' None + 'True' True case insensitive + 'False' False case insensitive + '0', '-0' 0 + 0xN, -0xN int from hex (postitive) (N is any number) + 0bN, -0bN int from binary (positive) (N is any number) + * try conversion to int, float, complex, fallback value + + """ + if len(value) == 0: + return '' + if value == 'None': + return None + lowered_value = value.lower() + if lowered_value == 'true': + return True + if lowered_value == 'false': + return False + valueneg = value[1:] if value[0] == '-' else value + if valueneg == '0': + return 0 + if valueneg == '': + return value + if valueneg[0] == '0': + if valueneg[1] in 'xX': + return int(value, 16) + elif valueneg[1] in 'bB': + return int(value, 2) + else: + try: + return int(value, 8) + except ValueError: + pass + try: + return int(value) + except ValueError: + pass + try: + return float(value) + except ValueError: + pass + try: + return complex(value) + except ValueError: + return value + + +def dict_from_dotted_str(items): + """parse multi dot-separated argument into dict. + EBS boot uses multi dot-separeted arguments like + BlockDeviceMapping.1.DeviceName=snap-id + Convert the above into + {'block_device_mapping': {'1': {'device_name': snap-id}}} + """ + args = {} + for key, value in items: + parts = key.split(".") + key = camelcase_to_underscore(parts[0]) + if isinstance(value, str) or isinstance(value, unicode): + # NOTE(vish): Automatically convert strings back + # into their respective values + value = _try_convert(value) + + if len(parts) > 1: + d = args.get(key, {}) + args[key] = d + for k in parts[1:-1]: + k = camelcase_to_underscore(k) + v = d.get(k, {}) + d[k] = v + d = v + d[camelcase_to_underscore(parts[-1])] = value + else: + args[key] = value + + return args diff --git a/nova/api/ec2/metadatarequesthandler.py b/nova/api/ec2/metadatarequesthandler.py index 28f99b0ef..b70266a20 100644 --- a/nova/api/ec2/metadatarequesthandler.py +++ b/nova/api/ec2/metadatarequesthandler.py @@ -23,6 +23,7 @@ import webob.exc from nova import log as logging from nova import flags +from nova import utils from nova import wsgi from nova.api.ec2 import cloud @@ -71,7 +72,15 @@ class MetadataRequestHandler(wsgi.Application): remote_address = req.remote_addr if FLAGS.use_forwarded_for: remote_address = req.headers.get('X-Forwarded-For', remote_address) - meta_data = cc.get_metadata(remote_address) + try: + meta_data = cc.get_metadata(remote_address) + except Exception: + LOG.exception(_('Failed to get metadata for ip: %s'), + remote_address) + msg = _('An unknown error has occurred. ' + 'Please try your request again.') + exc = webob.exc.HTTPInternalServerError(explanation=unicode(msg)) + return exc if meta_data is None: LOG.error(_('Failed to get metadata for ip: %s'), remote_address) raise webob.exc.HTTPNotFound() diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 5b7f080ad..f24017df0 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -26,7 +26,7 @@ import webob.exc from nova import flags from nova import log as logging -from nova import wsgi +from nova import wsgi as base_wsgi from nova.api.openstack import accounts from nova.api.openstack import faults from nova.api.openstack import backup_schedules @@ -40,6 +40,7 @@ from nova.api.openstack import servers from nova.api.openstack import server_metadata from nova.api.openstack import shared_ip_groups from nova.api.openstack import users +from nova.api.openstack import wsgi from nova.api.openstack import zones @@ -50,7 +51,7 @@ flags.DEFINE_bool('allow_admin_api', 'When True, this API service will accept admin operations.') -class FaultWrapper(wsgi.Middleware): +class FaultWrapper(base_wsgi.Middleware): """Calls down the middleware stack, making exceptions into faults.""" @webob.dec.wsgify(RequestClass=wsgi.Request) @@ -63,7 +64,7 @@ class FaultWrapper(wsgi.Middleware): return faults.Fault(exc) -class APIRouter(wsgi.Router): +class APIRouter(base_wsgi.Router): """ Routes requests on the OpenStack API to the appropriate controller and method. @@ -80,7 +81,9 @@ class APIRouter(wsgi.Router): self._setup_routes(mapper) super(APIRouter, self).__init__(mapper) - def _setup_routes(self, mapper): + def _setup_routes(self, mapper, version): + """Routes common to all versions.""" + server_members = self.server_members server_members['action'] = 'POST' if FLAGS.allow_admin_api: @@ -97,56 +100,64 @@ class APIRouter(wsgi.Router): server_members['reset_network'] = 'POST' server_members['inject_network_info'] = 'POST' - mapper.resource("zone", "zones", controller=zones.Controller(), - collection={'detail': 'GET', 'info': 'GET', - 'select': 'GET'}) - - mapper.resource("user", "users", controller=users.Controller(), + mapper.resource("user", "users", + controller=users.create_resource(), collection={'detail': 'GET'}) mapper.resource("account", "accounts", - controller=accounts.Controller(), + controller=accounts.create_resource(), collection={'detail': 'GET'}) - mapper.resource("console", "consoles", - controller=consoles.Controller(), - parent_resource=dict(member_name='server', - collection_name='servers')) - - super(APIRouter, self).__init__(mapper) + mapper.resource("zone", "zones", + controller=zones.create_resource(version), + collection={'detail': 'GET', + 'info': 'GET', + 'select': 'POST', + 'boot': 'POST'}) + mapper.resource("console", "consoles", + controller=consoles.create_resource(), + parent_resource=dict(member_name='server', + collection_name='servers')) -class APIRouterV10(APIRouter): - """Define routes specific to OpenStack API V1.0.""" - - def _setup_routes(self, mapper): - super(APIRouterV10, self)._setup_routes(mapper) mapper.resource("server", "servers", - controller=servers.ControllerV10(), + controller=servers.create_resource(version), collection={'detail': 'GET'}, member=self.server_members) mapper.resource("image", "images", - controller=images.ControllerV10(), + controller=images.create_resource(version), collection={'detail': 'GET'}) + mapper.resource("limit", "limits", + controller=limits.create_resource(version)) + mapper.resource("flavor", "flavors", - controller=flavors.ControllerV10(), + controller=flavors.create_resource(version), + collection={'detail': 'GET'}) + + super(APIRouter, self).__init__(mapper) + + +class APIRouterV10(APIRouter): + """Define routes specific to OpenStack API V1.0.""" + + def _setup_routes(self, mapper): + super(APIRouterV10, self)._setup_routes(mapper, '1.0') + mapper.resource("image", "images", + controller=images.create_resource('1.0'), collection={'detail': 'GET'}) mapper.resource("shared_ip_group", "shared_ip_groups", collection={'detail': 'GET'}, - controller=shared_ip_groups.Controller()) + controller=shared_ip_groups.create_resource()) mapper.resource("backup_schedule", "backup_schedule", - controller=backup_schedules.Controller(), + controller=backup_schedules.create_resource(), parent_resource=dict(member_name='server', collection_name='servers')) - mapper.resource("limit", "limits", - controller=limits.LimitsControllerV10()) - - mapper.resource("ip", "ips", controller=ips.Controller(), + mapper.resource("ip", "ips", controller=ips.create_resource(), collection=dict(public='GET', private='GET'), parent_resource=dict(member_name='server', collection_name='servers')) @@ -156,29 +167,13 @@ class APIRouterV11(APIRouter): """Define routes specific to OpenStack API V1.1.""" def _setup_routes(self, mapper): - super(APIRouterV11, self)._setup_routes(mapper) - mapper.resource("server", "servers", - controller=servers.ControllerV11(), - collection={'detail': 'GET'}, - member=self.server_members) - - mapper.resource("image", "images", - controller=images.ControllerV11(), - collection={'detail': 'GET'}) - + super(APIRouterV11, self)._setup_routes(mapper, '1.1') mapper.resource("image_meta", "meta", - controller=image_metadata.Controller(), + controller=image_metadata.create_resource(), parent_resource=dict(member_name='image', collection_name='images')) mapper.resource("server_meta", "meta", - controller=server_metadata.Controller(), + controller=server_metadata.create_resource(), parent_resource=dict(member_name='server', collection_name='servers')) - - mapper.resource("flavor", "flavors", - controller=flavors.ControllerV11(), - collection={'detail': 'GET'}) - - mapper.resource("limit", "limits", - controller=limits.LimitsControllerV11()) diff --git a/nova/api/openstack/accounts.py b/nova/api/openstack/accounts.py index 00fdd4540..0dcd37217 100644 --- a/nova/api/openstack/accounts.py +++ b/nova/api/openstack/accounts.py @@ -20,8 +20,9 @@ from nova import flags from nova import log as logging from nova.auth import manager -from nova.api.openstack import common from nova.api.openstack import faults +from nova.api.openstack import wsgi + FLAGS = flags.FLAGS LOG = logging.getLogger('nova.api.openstack') @@ -34,12 +35,7 @@ def _translate_keys(account): manager=account.project_manager_id) -class Controller(common.OpenstackController): - - _serialization_metadata = { - 'application/xml': { - "attributes": { - "account": ["id", "name", "description", "manager"]}}} +class Controller(object): def __init__(self): self.manager = manager.AuthManager() @@ -66,20 +62,33 @@ class Controller(common.OpenstackController): self.manager.delete_project(id) return {} - def create(self, req): + def create(self, req, body): """We use update with create-or-update semantics because the id comes from an external source""" raise faults.Fault(webob.exc.HTTPNotImplemented()) - def update(self, req, id): + def update(self, req, id, body): """This is really create or update.""" self._check_admin(req.environ['nova.context']) - env = self._deserialize(req.body, req.get_content_type()) - description = env['account'].get('description') - manager = env['account'].get('manager') + description = body['account'].get('description') + manager = body['account'].get('manager') try: account = self.manager.get_project(id) self.manager.modify_project(id, manager, description) except exception.NotFound: account = self.manager.create_project(id, manager, description) return dict(account=_translate_keys(account)) + + +def create_resource(): + metadata = { + "attributes": { + "account": ["id", "name", "description", "manager"], + }, + } + + serializers = { + 'application/xml': wsgi.XMLDictSerializer(metadata=metadata), + } + + return wsgi.Resource(Controller(), serializers=serializers) diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 6c6ee22a2..7c3e683d6 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -13,9 +13,8 @@ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations -# under the License.import datetime +# under the License. -import datetime import hashlib import time @@ -50,19 +49,22 @@ class AuthMiddleware(wsgi.Middleware): if not self.has_authentication(req): return self.authenticate(req) user = self.get_user_by_authentication(req) - accounts = self.auth.get_projects(user=user) if not user: token = req.headers["X-Auth-Token"] msg = _("%(user)s could not be found with token '%(token)s'") LOG.warn(msg % locals()) return faults.Fault(webob.exc.HTTPUnauthorized()) - if accounts: - #we are punting on this til auth is settled, - #and possibly til api v1.1 (mdragon) - account = accounts[0] - else: - return faults.Fault(webob.exc.HTTPUnauthorized()) + try: + account = req.headers["X-Auth-Project-Id"] + except KeyError: + # FIXME(usrleon): It needed only for compatibility + # while osapi clients don't use this header + accounts = self.auth.get_projects(user=user) + if accounts: + account = accounts[0] + else: + return faults.Fault(webob.exc.HTTPUnauthorized()) if not self.auth.is_admin(user) and \ not self.auth.is_project_member(user, account): @@ -127,7 +129,7 @@ class AuthMiddleware(wsgi.Middleware): except exception.NotFound: return None if token: - delta = datetime.datetime.utcnow() - token['created_at'] + delta = utils.utcnow() - token['created_at'] if delta.days >= 2: self.db.auth_token_destroy(ctxt, token['token_hash']) else: diff --git a/nova/api/openstack/backup_schedules.py b/nova/api/openstack/backup_schedules.py index 4bf744046..71a14d4ce 100644 --- a/nova/api/openstack/backup_schedules.py +++ b/nova/api/openstack/backup_schedules.py @@ -19,9 +19,8 @@ import time from webob import exc -from nova.api.openstack import common from nova.api.openstack import faults -import nova.image.service +from nova.api.openstack import wsgi def _translate_keys(inst): @@ -29,14 +28,9 @@ def _translate_keys(inst): return dict(backupSchedule=inst) -class Controller(common.OpenstackController): +class Controller(object): """ The backup schedule API controller for the Openstack API """ - _serialization_metadata = { - 'application/xml': { - 'attributes': { - 'backupSchedule': []}}} - def __init__(self): pass @@ -48,7 +42,7 @@ class Controller(common.OpenstackController): """ Returns a single backup schedule for a given instance """ return faults.Fault(exc.HTTPNotImplemented()) - def create(self, req, server_id): + def create(self, req, server_id, body): """ No actual update method required, since the existing API allows both create and update through a POST """ return faults.Fault(exc.HTTPNotImplemented()) @@ -56,3 +50,18 @@ class Controller(common.OpenstackController): def delete(self, req, server_id, id): """ Deletes an existing backup schedule """ return faults.Fault(exc.HTTPNotImplemented()) + + +def create_resource(): + metadata = { + 'attributes': { + 'backupSchedule': [], + }, + } + + serializers = { + 'application/xml': wsgi.XMLDictSerializer(xmlns=wsgi.XMLNS_V10, + metadata=metadata), + } + + return wsgi.Resource(Controller(), serializers=serializers) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 32cd689ca..4da7ec0ef 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -23,12 +23,9 @@ import webob from nova import exception from nova import flags from nova import log as logging -from nova import wsgi LOG = logging.getLogger('nova.api.openstack.common') - - FLAGS = flags.FLAGS @@ -36,6 +33,37 @@ XML_NS_V10 = 'http://docs.rackspacecloud.com/servers/api/v1.0' XML_NS_V11 = 'http://docs.openstack.org/compute/api/v1.1' +def get_pagination_params(request): + """Return marker, limit tuple from request. + + :param request: `wsgi.Request` possibly containing 'marker' and 'limit' + GET variables. 'marker' is the id of the last element + the client has seen, and 'limit' is the maximum number + of items to return. If 'limit' is not specified, 0, or + > max_limit, we default to max_limit. Negative values + for either marker or limit will cause + exc.HTTPBadRequest() exceptions to be raised. + + """ + try: + marker = int(request.GET.get('marker', 0)) + except ValueError: + raise webob.exc.HTTPBadRequest(_('marker param must be an integer')) + + try: + limit = int(request.GET.get('limit', 0)) + except ValueError: + raise webob.exc.HTTPBadRequest(_('limit param must be an integer')) + + if limit < 0: + raise webob.exc.HTTPBadRequest(_('limit param must be positive')) + + if marker < 0: + raise webob.exc.HTTPBadRequest(_('marker param must be positive')) + + return(marker, limit) + + def limited(items, request, max_limit=FLAGS.osapi_max_limit): """ Return a slice of items according to requested offset and limit. @@ -72,19 +100,10 @@ def limited(items, request, max_limit=FLAGS.osapi_max_limit): def limited_by_marker(items, request, max_limit=FLAGS.osapi_max_limit): """Return a slice of items according to the requested marker and limit.""" + (marker, limit) = get_pagination_params(request) - try: - marker = int(request.GET.get('marker', 0)) - except ValueError: - raise webob.exc.HTTPBadRequest(_('marker param must be an integer')) - - try: - limit = int(request.GET.get('limit', max_limit)) - except ValueError: - raise webob.exc.HTTPBadRequest(_('limit param must be an integer')) - - if limit < 0: - raise webob.exc.HTTPBadRequest(_('limit param must be positive')) + if limit == 0: + limit = max_limit limit = min(max_limit, limit) start_index = 0 @@ -100,34 +119,6 @@ def limited_by_marker(items, request, max_limit=FLAGS.osapi_max_limit): return items[start_index:range_end] -def get_image_id_from_image_hash(image_service, context, image_hash): - """Given an Image ID Hash, return an objectstore Image ID. - - image_service - reference to objectstore compatible image service. - context - security context for image service requests. - image_hash - hash of the image ID. - """ - - # FIX(sandy): This is terribly inefficient. It pulls all images - # from objectstore in order to find the match. ObjectStore - # should have a numeric counterpart to the string ID. - try: - items = image_service.detail(context) - except NotImplementedError: - items = image_service.index(context) - for image in items: - image_id = image['id'] - try: - if abs(hash(image_id)) == int(image_hash): - return image_id - except ValueError: - msg = _("Requested image_id has wrong format: %s," - "should have numerical format") % image_id - LOG.error(msg) - raise Exception(msg) - raise exception.ImageNotFound(image_id=image_hash) - - def get_id_from_href(href): """Return the id portion of a url as an int. @@ -146,9 +137,3 @@ def get_id_from_href(href): except: LOG.debug(_("Error extracting id from href: %s") % href) raise webob.exc.HTTPBadRequest(_('could not parse id from href')) - - -class OpenstackController(wsgi.Controller): - def get_default_xmlns(self, req): - # Use V10 by default - return XML_NS_V10 diff --git a/nova/api/openstack/consoles.py b/nova/api/openstack/consoles.py index 1a77f25d7..bccf04d8f 100644 --- a/nova/api/openstack/consoles.py +++ b/nova/api/openstack/consoles.py @@ -19,8 +19,8 @@ from webob import exc from nova import console from nova import exception -from nova.api.openstack import common from nova.api.openstack import faults +from nova.api.openstack import wsgi def _translate_keys(cons): @@ -43,17 +43,11 @@ def _translate_detail_keys(cons): return dict(console=info) -class Controller(common.OpenstackController): - """The Consoles Controller for the Openstack API""" - - _serialization_metadata = { - 'application/xml': { - 'attributes': { - 'console': []}}} +class Controller(object): + """The Consoles controller for the Openstack API""" def __init__(self): self.console_api = console.API() - super(Controller, self).__init__() def index(self, req, server_id): """Returns a list of consoles for this instance""" @@ -63,9 +57,8 @@ class Controller(common.OpenstackController): return dict(consoles=[_translate_keys(console) for console in consoles]) - def create(self, req, server_id): + def create(self, req, server_id, body): """Creates a new console""" - #info = self._deserialize(req.body, req.get_content_type()) self.console_api.create_console( req.environ['nova.context'], int(server_id)) @@ -94,3 +87,17 @@ class Controller(common.OpenstackController): except exception.NotFound: return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() + + +def create_resource(): + metadata = { + 'attributes': { + 'console': [], + }, + } + + serializers = { + 'application/xml': wsgi.XMLDictSerializer(metadata=metadata), + } + + return wsgi.Resource(Controller(), serializers=serializers) diff --git a/nova/api/openstack/contrib/__init__.py b/nova/api/openstack/contrib/__init__.py index b42a1d89d..acb5eb280 100644 --- a/nova/api/openstack/contrib/__init__.py +++ b/nova/api/openstack/contrib/__init__.py @@ -13,7 +13,7 @@ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations -# under the License.import datetime +# under the License. """Contrib contains extensions that are shipped with nova. diff --git a/nova/api/openstack/contrib/volumes.py b/nova/api/openstack/contrib/volumes.py index 18de2ec71..e5e2c5b50 100644 --- a/nova/api/openstack/contrib/volumes.py +++ b/nova/api/openstack/contrib/volumes.py @@ -22,7 +22,6 @@ from nova import exception from nova import flags from nova import log as logging from nova import volume -from nova import wsgi from nova.api.openstack import common from nova.api.openstack import extensions from nova.api.openstack import faults @@ -64,7 +63,7 @@ def _translate_volume_summary_view(context, vol): return d -class VolumeController(wsgi.Controller): +class VolumeController(object): """The Volumes API controller for the OpenStack API.""" _serialization_metadata = { @@ -124,18 +123,17 @@ class VolumeController(wsgi.Controller): res = [entity_maker(context, vol) for vol in limited_list] return {'volumes': res} - def create(self, req): + def create(self, req, body): """Creates a new volume.""" context = req.environ['nova.context'] - env = self._deserialize(req.body, req.get_content_type()) - if not env: + if not body: return faults.Fault(exc.HTTPUnprocessableEntity()) - vol = env['volume'] + vol = body['volume'] size = vol['size'] LOG.audit(_("Create volume of %s GB"), size, context=context) - new_volume = self.volume_api.create(context, size, + new_volume = self.volume_api.create(context, size, None, vol.get('display_name'), vol.get('display_description')) @@ -175,7 +173,7 @@ def _translate_attachment_summary_view(_context, vol): return d -class VolumeAttachmentController(wsgi.Controller): +class VolumeAttachmentController(object): """The volume attachment API controller for the Openstack API. A child resource of the server. Note that we use the volume id @@ -219,17 +217,16 @@ class VolumeAttachmentController(wsgi.Controller): return {'volumeAttachment': _translate_attachment_detail_view(context, vol)} - def create(self, req, server_id): + def create(self, req, server_id, body): """Attach a volume to an instance.""" context = req.environ['nova.context'] - env = self._deserialize(req.body, req.get_content_type()) - if not env: + if not body: return faults.Fault(exc.HTTPUnprocessableEntity()) instance_id = server_id - volume_id = env['volumeAttachment']['volumeId'] - device = env['volumeAttachment']['device'] + volume_id = body['volumeAttachment']['volumeId'] + device = body['volumeAttachment']['device'] msg = _("Attach volume %(volume_id)s to instance %(server_id)s" " at %(device)s") % locals() @@ -259,7 +256,7 @@ class VolumeAttachmentController(wsgi.Controller): # TODO(justinsb): How do I return "accepted" here? return {'volumeAttachment': attachment} - def update(self, _req, _server_id, _id): + def update(self, req, server_id, id, body): """Update a volume attachment. We don't currently support this.""" return faults.Fault(exc.HTTPBadRequest()) @@ -304,7 +301,7 @@ class Volumes(extensions.ExtensionDescriptor): return "Volumes" def get_alias(self): - return "VOLUMES" + return "os-volumes" def get_description(self): return "Volumes support" @@ -320,12 +317,12 @@ class Volumes(extensions.ExtensionDescriptor): # NOTE(justinsb): No way to provide singular name ('volume') # Does this matter? - res = extensions.ResourceExtension('volumes', + res = extensions.ResourceExtension('os-volumes', VolumeController(), collection_actions={'detail': 'GET'}) resources.append(res) - res = extensions.ResourceExtension('volume_attachments', + res = extensions.ResourceExtension('os-volume_attachments', VolumeAttachmentController(), parent=dict( member_name='server', diff --git a/nova/api/openstack/create_instance_helper.py b/nova/api/openstack/create_instance_helper.py new file mode 100644 index 000000000..436e524c1 --- /dev/null +++ b/nova/api/openstack/create_instance_helper.py @@ -0,0 +1,343 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import base64 +import re +import webob + +from webob import exc +from xml.dom import minidom + +from nova import exception +from nova import flags +from nova import log as logging +import nova.image +from nova import quota +from nova import utils + +from nova.compute import instance_types +from nova.api.openstack import faults +from nova.api.openstack import wsgi +from nova.auth import manager as auth_manager + + +LOG = logging.getLogger('nova.api.openstack.create_instance_helper') +FLAGS = flags.FLAGS + + +class CreateFault(exception.NovaException): + message = _("Invalid parameters given to create_instance.") + + def __init__(self, fault): + self.fault = fault + super(CreateFault, self).__init__() + + +class CreateInstanceHelper(object): + """This is the base class for OS API Controllers that + are capable of creating instances (currently Servers and Zones). + + Once we stabilize the Zones portion of the API we may be able + to move this code back into servers.py + """ + + def __init__(self, controller): + """We need the image service to create an instance.""" + self.controller = controller + self._image_service = utils.import_object(FLAGS.image_service) + super(CreateInstanceHelper, self).__init__() + + def create_instance(self, req, body, create_method): + """Creates a new server for the given user. The approach + used depends on the create_method. For example, the standard + POST /server call uses compute.api.create(), while + POST /zones/server uses compute.api.create_all_at_once(). + + The problem is, both approaches return different values (i.e. + [instance dicts] vs. reservation_id). So the handling of the + return type from this method is left to the caller. + """ + if not body: + raise faults.Fault(exc.HTTPUnprocessableEntity()) + + context = req.environ['nova.context'] + + password = self.controller._get_server_admin_password(body['server']) + + key_name = None + key_data = None + key_pairs = auth_manager.AuthManager.get_key_pairs(context) + if key_pairs: + key_pair = key_pairs[0] + key_name = key_pair['name'] + key_data = key_pair['public_key'] + + image_href = self.controller._image_ref_from_req_data(body) + try: + image_service, image_id = nova.image.get_image_service(image_href) + kernel_id, ramdisk_id = self._get_kernel_ramdisk_from_image( + req, image_id) + images = set([str(x['id']) for x in image_service.index(context)]) + assert str(image_id) in images + except Exception, e: + msg = _("Cannot find requested image %(image_href)s: %(e)s" % + locals()) + raise faults.Fault(exc.HTTPBadRequest(explanation=msg)) + + personality = body['server'].get('personality') + + injected_files = [] + if personality: + injected_files = self._get_injected_files(personality) + + flavor_id = self.controller._flavor_id_from_req_data(body) + + if not 'name' in body['server']: + msg = _("Server name is not defined") + raise exc.HTTPBadRequest(explanation=msg) + + zone_blob = body['server'].get('blob') + name = body['server']['name'] + self._validate_server_name(name) + name = name.strip() + + reservation_id = body['server'].get('reservation_id') + + try: + inst_type = \ + instance_types.get_instance_type_by_flavor_id(flavor_id) + extra_values = { + 'instance_type': inst_type, + 'image_ref': image_href, + 'password': password} + + return (extra_values, + create_method(context, + inst_type, + image_id, + kernel_id=kernel_id, + ramdisk_id=ramdisk_id, + display_name=name, + display_description=name, + key_name=key_name, + key_data=key_data, + metadata=body['server'].get('metadata', {}), + injected_files=injected_files, + admin_password=password, + zone_blob=zone_blob, + reservation_id=reservation_id)) + except quota.QuotaError as error: + self._handle_quota_error(error) + except exception.ImageNotFound as error: + msg = _("Can not find requested image") + raise faults.Fault(exc.HTTPBadRequest(explanation=msg)) + + # Let the caller deal with unhandled exceptions. + + def _handle_quota_error(self, error): + """ + Reraise quota errors as api-specific http exceptions + """ + if error.code == "OnsetFileLimitExceeded": + expl = _("Personality file limit exceeded") + raise exc.HTTPBadRequest(explanation=expl) + if error.code == "OnsetFilePathLimitExceeded": + expl = _("Personality file path too long") + raise exc.HTTPBadRequest(explanation=expl) + if error.code == "OnsetFileContentLimitExceeded": + expl = _("Personality file content too long") + raise exc.HTTPBadRequest(explanation=expl) + # if the original error is okay, just reraise it + raise error + + def _deserialize_create(self, request): + """ + Deserialize a create request + + Overrides normal behavior in the case of xml content + """ + if request.content_type == "application/xml": + deserializer = ServerCreateRequestXMLDeserializer() + return deserializer.deserialize(request.body) + else: + return self._deserialize(request.body, request.get_content_type()) + + def _validate_server_name(self, value): + if not isinstance(value, basestring): + msg = _("Server name is not a string or unicode") + raise exc.HTTPBadRequest(explanation=msg) + + if value.strip() == '': + msg = _("Server name is an empty string") + raise exc.HTTPBadRequest(explanation=msg) + + def _get_kernel_ramdisk_from_image(self, req, image_id): + """Fetch an image from the ImageService, then if present, return the + associated kernel and ramdisk image IDs. + """ + context = req.environ['nova.context'] + image_meta = self._image_service.show(context, image_id) + # NOTE(sirp): extracted to a separate method to aid unit-testing, the + # new method doesn't need a request obj or an ImageService stub + kernel_id, ramdisk_id = self._do_get_kernel_ramdisk_from_image( + image_meta) + return kernel_id, ramdisk_id + + @staticmethod + def _do_get_kernel_ramdisk_from_image(image_meta): + """Given an ImageService image_meta, return kernel and ramdisk image + ids if present. + + This is only valid for `ami` style images. + """ + image_id = image_meta['id'] + if image_meta['status'] != 'active': + raise exception.ImageUnacceptable(image_id=image_id, + reason=_("status is not active")) + + if image_meta.get('container_format') != 'ami': + return None, None + + try: + kernel_id = image_meta['properties']['kernel_id'] + except KeyError: + raise exception.KernelNotFoundForImage(image_id=image_id) + + try: + ramdisk_id = image_meta['properties']['ramdisk_id'] + except KeyError: + raise exception.RamdiskNotFoundForImage(image_id=image_id) + + return kernel_id, ramdisk_id + + def _get_injected_files(self, personality): + """ + Create a list of injected files from the personality attribute + + At this time, injected_files must be formatted as a list of + (file_path, file_content) pairs for compatibility with the + underlying compute service. + """ + injected_files = [] + + for item in personality: + try: + path = item['path'] + contents = item['contents'] + except KeyError as key: + expl = _('Bad personality format: missing %s') % key + raise exc.HTTPBadRequest(explanation=expl) + except TypeError: + expl = _('Bad personality format') + raise exc.HTTPBadRequest(explanation=expl) + try: + contents = base64.b64decode(contents) + except TypeError: + expl = _('Personality content for %s cannot be decoded') % path + raise exc.HTTPBadRequest(explanation=expl) + injected_files.append((path, contents)) + return injected_files + + def _get_server_admin_password_old_style(self, server): + """ Determine the admin password for a server on creation """ + return utils.generate_password(16) + + def _get_server_admin_password_new_style(self, server): + """ Determine the admin password for a server on creation """ + password = server.get('adminPass') + + if password is None: + return utils.generate_password(16) + if not isinstance(password, basestring) or password == '': + msg = _("Invalid adminPass") + raise exc.HTTPBadRequest(explanation=msg) + return password + + +class ServerXMLDeserializer(wsgi.XMLDeserializer): + """ + Deserializer to handle xml-formatted server create requests. + + Handles standard server attributes as well as optional metadata + and personality attributes + """ + + def create(self, string): + """Deserialize an xml-formatted server create request""" + dom = minidom.parseString(string) + server = self._extract_server(dom) + return {'server': server} + + def _extract_server(self, node): + """Marshal the server attribute of a parsed request""" + server = {} + server_node = self._find_first_child_named(node, 'server') + for attr in ["name", "imageId", "flavorId", "imageRef", "flavorRef"]: + if server_node.getAttribute(attr): + server[attr] = server_node.getAttribute(attr) + metadata = self._extract_metadata(server_node) + if metadata is not None: + server["metadata"] = metadata + personality = self._extract_personality(server_node) + if personality is not None: + server["personality"] = personality + return server + + def _extract_metadata(self, server_node): + """Marshal the metadata attribute of a parsed request""" + metadata_node = self._find_first_child_named(server_node, "metadata") + if metadata_node is None: + return None + metadata = {} + for meta_node in self._find_children_named(metadata_node, "meta"): + key = meta_node.getAttribute("key") + metadata[key] = self._extract_text(meta_node) + return metadata + + def _extract_personality(self, server_node): + """Marshal the personality attribute of a parsed request""" + personality_node = \ + self._find_first_child_named(server_node, "personality") + if personality_node is None: + return None + personality = [] + for file_node in self._find_children_named(personality_node, "file"): + item = {} + if file_node.hasAttribute("path"): + item["path"] = file_node.getAttribute("path") + item["contents"] = self._extract_text(file_node) + personality.append(item) + return personality + + def _find_first_child_named(self, parent, name): + """Search a nodes children for the first child with a given name""" + for node in parent.childNodes: + if node.nodeName == name: + return node + return None + + def _find_children_named(self, parent, name): + """Return all of a nodes children who have the given name""" + for node in parent.childNodes: + if node.nodeName == name: + yield node + + def _extract_text(self, node): + """Get the text field contained by the given node""" + if len(node.childNodes) == 1: + child = node.childNodes[0] + if child.nodeType == child.TEXT_NODE: + return child.nodeValue + return "" diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index 8e77b25fb..da06ecd15 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -27,9 +27,10 @@ import webob.exc from nova import exception from nova import flags from nova import log as logging -from nova import wsgi +from nova import wsgi as base_wsgi from nova.api.openstack import common from nova.api.openstack import faults +from nova.api.openstack import wsgi LOG = logging.getLogger('extensions') @@ -115,28 +116,34 @@ class ExtensionDescriptor(object): return request_exts -class ActionExtensionController(common.OpenstackController): - +class ActionExtensionController(object): def __init__(self, application): - self.application = application self.action_handlers = {} def add_action(self, action_name, handler): self.action_handlers[action_name] = handler - def action(self, req, id): - - input_dict = self._deserialize(req.body, req.get_content_type()) + def action(self, req, id, body): for action_name, handler in self.action_handlers.iteritems(): - if action_name in input_dict: - return handler(input_dict, req, id) + if action_name in body: + return handler(body, req, id) # no action handler found (bump to downstream application) res = self.application return res -class RequestExtensionController(common.OpenstackController): +class ActionExtensionResource(wsgi.Resource): + + def __init__(self, application): + controller = ActionExtensionController(application) + wsgi.Resource.__init__(self, controller) + + def add_action(self, action_name, handler): + self.controller.add_action(action_name, handler) + + +class RequestExtensionController(object): def __init__(self, application): self.application = application @@ -153,7 +160,17 @@ class RequestExtensionController(common.OpenstackController): return res -class ExtensionController(common.OpenstackController): +class RequestExtensionResource(wsgi.Resource): + + def __init__(self, application): + controller = RequestExtensionController(application) + wsgi.Resource.__init__(self, controller) + + def add_handler(self, handler): + self.controller.add_handler(handler) + + +class ExtensionsResource(wsgi.Resource): def __init__(self, extension_manager): self.extension_manager = extension_manager @@ -186,7 +203,7 @@ class ExtensionController(common.OpenstackController): raise faults.Fault(webob.exc.HTTPNotFound()) -class ExtensionMiddleware(wsgi.Middleware): +class ExtensionMiddleware(base_wsgi.Middleware): """Extensions middleware for WSGI.""" @classmethod def factory(cls, global_config, **local_config): @@ -195,43 +212,43 @@ class ExtensionMiddleware(wsgi.Middleware): return cls(app, **local_config) return _factory - def _action_ext_controllers(self, application, ext_mgr, mapper): - """Return a dict of ActionExtensionController-s by collection.""" - action_controllers = {} + def _action_ext_resources(self, application, ext_mgr, mapper): + """Return a dict of ActionExtensionResource-s by collection.""" + action_resources = {} for action in ext_mgr.get_actions(): - if not action.collection in action_controllers.keys(): - controller = ActionExtensionController(application) + if not action.collection in action_resources.keys(): + resource = ActionExtensionResource(application) mapper.connect("/%s/:(id)/action.:(format)" % action.collection, action='action', - controller=controller, + controller=resource, conditions=dict(method=['POST'])) mapper.connect("/%s/:(id)/action" % action.collection, action='action', - controller=controller, + controller=resource, conditions=dict(method=['POST'])) - action_controllers[action.collection] = controller + action_resources[action.collection] = resource - return action_controllers + return action_resources - def _request_ext_controllers(self, application, ext_mgr, mapper): - """Returns a dict of RequestExtensionController-s by collection.""" - request_ext_controllers = {} + def _request_ext_resources(self, application, ext_mgr, mapper): + """Returns a dict of RequestExtensionResource-s by collection.""" + request_ext_resources = {} for req_ext in ext_mgr.get_request_extensions(): - if not req_ext.key in request_ext_controllers.keys(): - controller = RequestExtensionController(application) + if not req_ext.key in request_ext_resources.keys(): + resource = RequestExtensionResource(application) mapper.connect(req_ext.url_route + '.:(format)', action='process', - controller=controller, + controller=resource, conditions=req_ext.conditions) mapper.connect(req_ext.url_route, action='process', - controller=controller, + controller=resource, conditions=req_ext.conditions) - request_ext_controllers[req_ext.key] = controller + request_ext_resources[req_ext.key] = resource - return request_ext_controllers + return request_ext_resources def __init__(self, application, ext_mgr=None): @@ -246,22 +263,22 @@ class ExtensionMiddleware(wsgi.Middleware): LOG.debug(_('Extended resource: %s'), resource.collection) mapper.resource(resource.collection, resource.collection, - controller=resource.controller, + controller=wsgi.Resource(resource.controller), collection=resource.collection_actions, member=resource.member_actions, parent_resource=resource.parent) # extended actions - action_controllers = self._action_ext_controllers(application, ext_mgr, + action_resources = self._action_ext_resources(application, ext_mgr, mapper) for action in ext_mgr.get_actions(): LOG.debug(_('Extended action: %s'), action.action_name) - controller = action_controllers[action.collection] - controller.add_action(action.action_name, action.handler) + resource = action_resources[action.collection] + resource.add_action(action.action_name, action.handler) # extended requests - req_controllers = self._request_ext_controllers(application, ext_mgr, - mapper) + req_controllers = self._request_ext_resources(application, ext_mgr, + mapper) for request_ext in ext_mgr.get_request_extensions(): LOG.debug(_('Extended request: %s'), request_ext.key) controller = req_controllers[request_ext.key] @@ -313,7 +330,7 @@ class ExtensionManager(object): """Returns a list of ResourceExtension objects.""" resources = [] resources.append(ResourceExtension('extensions', - ExtensionController(self))) + ExtensionsResource(self))) for alias, ext in self.extensions.iteritems(): try: resources.extend(ext.get_resources()) @@ -357,6 +374,8 @@ class ExtensionManager(object): LOG.debug(_('Ext updated: %s'), extension.get_updated()) except AttributeError as ex: LOG.exception(_("Exception loading extension: %s"), unicode(ex)) + return False + return True def _load_all_extensions(self): """Load extensions from the configured path. @@ -395,22 +414,23 @@ class ExtensionManager(object): 'file': ext_path}) continue new_ext = new_ext_class() - self._check_extension(new_ext) - self._add_extension(new_ext) + self.add_extension(new_ext) + + def add_extension(self, ext): + # Do nothing if the extension doesn't check out + if not self._check_extension(ext): + return - def _add_extension(self, ext): alias = ext.get_alias() LOG.audit(_('Loaded extension: %s'), alias) - self._check_extension(ext) - if alias in self.extensions: raise exception.Error("Found duplicate extension: %s" % alias) self.extensions[alias] = ext class RequestExtension(object): - """Extend requests and responses of core nova OpenStack API controllers. + """Extend requests and responses of core nova OpenStack API resources. Provide a way to add data to responses and handle custom request data that is sent to core nova OpenStack API controllers. @@ -424,7 +444,7 @@ class RequestExtension(object): class ActionExtension(object): - """Add custom actions to core nova OpenStack API controllers.""" + """Add custom actions to core nova OpenStack API resources.""" def __init__(self, collection, action_name, handler): self.collection = collection diff --git a/nova/api/openstack/faults.py b/nova/api/openstack/faults.py index 87118ce19..b9a23c126 100644 --- a/nova/api/openstack/faults.py +++ b/nova/api/openstack/faults.py @@ -19,8 +19,7 @@ import webob.dec import webob.exc -from nova import wsgi -from nova.api.openstack import common +from nova.api.openstack import wsgi class Fault(webob.exc.HTTPException): @@ -55,13 +54,21 @@ class Fault(webob.exc.HTTPException): if code == 413: retry = self.wrapped_exc.headers['Retry-After'] fault_data[fault_name]['retryAfter'] = retry + # 'code' is an attribute on the fault tag itself - metadata = {'application/xml': {'attributes': {fault_name: 'code'}}} - default_xmlns = common.XML_NS_V10 - serializer = wsgi.Serializer(metadata, default_xmlns) + metadata = {'attributes': {fault_name: 'code'}} + content_type = req.best_match_content_type() - self.wrapped_exc.body = serializer.serialize(fault_data, content_type) + + serializer = { + 'application/xml': wsgi.XMLDictSerializer(metadata=metadata, + xmlns=wsgi.XMLNS_V10), + 'application/json': wsgi.JSONDictSerializer(), + }[content_type] + + self.wrapped_exc.body = serializer.serialize(fault_data) self.wrapped_exc.content_type = content_type + return self.wrapped_exc @@ -70,14 +77,6 @@ class OverLimitFault(webob.exc.HTTPException): Rate-limited request response. """ - _serialization_metadata = { - "application/xml": { - "attributes": { - "overLimitFault": "code", - }, - }, - } - def __init__(self, message, details, retry_time): """ Initialize new `OverLimitFault` with relevant information. @@ -97,8 +96,16 @@ class OverLimitFault(webob.exc.HTTPException): Return the wrapped exception with a serialized body conforming to our error format. """ - serializer = wsgi.Serializer(self._serialization_metadata) content_type = request.best_match_content_type() - content = serializer.serialize(self.content, content_type) + metadata = {"attributes": {"overLimitFault": "code"}} + + serializer = { + 'application/xml': wsgi.XMLDictSerializer(metadata=metadata, + xmlns=wsgi.XMLNS_V10), + 'application/json': wsgi.JSONDictSerializer(), + }[content_type] + + content = serializer.serialize(self.content) self.wrapped_exc.body = content + return self.wrapped_exc diff --git a/nova/api/openstack/flavors.py b/nova/api/openstack/flavors.py index 4c5971cf6..a21ff6cb2 100644 --- a/nova/api/openstack/flavors.py +++ b/nova/api/openstack/flavors.py @@ -19,22 +19,13 @@ import webob from nova import db from nova import exception -from nova.api.openstack import common from nova.api.openstack import views +from nova.api.openstack import wsgi -class Controller(common.OpenstackController): +class Controller(object): """Flavor controller for the OpenStack API.""" - _serialization_metadata = { - 'application/xml': { - "attributes": { - "flavor": ["id", "name", "ram", "disk"], - "link": ["rel", "type", "href"], - } - } - } - def index(self, req): """Return all flavors in brief.""" items = self._get_flavors(req, is_detail=False) @@ -71,14 +62,31 @@ class Controller(common.OpenstackController): class ControllerV10(Controller): + def _get_view_builder(self, req): return views.flavors.ViewBuilder() class ControllerV11(Controller): + def _get_view_builder(self, req): base_url = req.application_url return views.flavors.ViewBuilderV11(base_url) - def get_default_xmlns(self, req): - return common.XML_NS_V11 + +def create_resource(version='1.0'): + controller = { + '1.0': ControllerV10, + '1.1': ControllerV11, + }[version]() + + xmlns = { + '1.0': wsgi.XMLNS_V10, + '1.1': wsgi.XMLNS_V11, + }[version] + + serializers = { + 'application/xml': wsgi.XMLDictSerializer(xmlns=xmlns), + } + + return wsgi.Resource(controller, serializers=serializers) diff --git a/nova/api/openstack/image_metadata.py b/nova/api/openstack/image_metadata.py index 1eccc0174..c0e92f2fc 100644 --- a/nova/api/openstack/image_metadata.py +++ b/nova/api/openstack/image_metadata.py @@ -16,24 +16,24 @@ # under the License. from webob import exc +from xml.dom import minidom from nova import flags +from nova import image from nova import quota from nova import utils -from nova import wsgi -from nova.api.openstack import common from nova.api.openstack import faults +from nova.api.openstack import wsgi FLAGS = flags.FLAGS -class Controller(common.OpenstackController): +class Controller(object): """The image metadata API controller for the Openstack API""" def __init__(self): - self.image_service = utils.import_object(FLAGS.image_service) - super(Controller, self).__init__() + self.image_service = image.get_default_image_service() def _get_metadata(self, context, image_id, image=None): if not image: @@ -60,13 +60,12 @@ class Controller(common.OpenstackController): context = req.environ['nova.context'] metadata = self._get_metadata(context, image_id) if id in metadata: - return {id: metadata[id]} + return {'meta': {id: metadata[id]}} else: return faults.Fault(exc.HTTPNotFound()) - def create(self, req, image_id): + def create(self, req, image_id, body): context = req.environ['nova.context'] - body = self._deserialize(req.body, req.get_content_type()) img = self.image_service.show(context, image_id) metadata = self._get_metadata(context, image_id, img) if 'metadata' in body: @@ -77,18 +76,24 @@ class Controller(common.OpenstackController): self.image_service.update(context, image_id, img, None) return dict(metadata=metadata) - def update(self, req, image_id, id): + def update(self, req, image_id, id, body): context = req.environ['nova.context'] - body = self._deserialize(req.body, req.get_content_type()) - if not id in body: + + try: + meta = body['meta'] + except KeyError: + expl = _('Incorrect request body format') + raise exc.HTTPBadRequest(explanation=expl) + + if not id in meta: expl = _('Request body and URI mismatch') raise exc.HTTPBadRequest(explanation=expl) - if len(body) > 1: + if len(meta) > 1: expl = _('Request body contains too many items') raise exc.HTTPBadRequest(explanation=expl) img = self.image_service.show(context, image_id) metadata = self._get_metadata(context, image_id, img) - metadata[id] = body[id] + metadata[id] = meta[id] self._check_quota_limit(context, metadata) img['properties'] = metadata self.image_service.update(context, image_id, img, None) @@ -104,3 +109,57 @@ class Controller(common.OpenstackController): metadata.pop(id) img['properties'] = metadata self.image_service.update(context, image_id, img, None) + + +class ImageMetadataXMLSerializer(wsgi.XMLDictSerializer): + def __init__(self): + xmlns = wsgi.XMLNS_V11 + super(ImageMetadataXMLSerializer, self).__init__(xmlns=xmlns) + + def _meta_item_to_xml(self, doc, key, value): + node = doc.createElement('meta') + node.setAttribute('key', key) + text = doc.createTextNode(value) + node.appendChild(text) + return node + + def _meta_list_to_xml(self, xml_doc, meta_items): + container_node = xml_doc.createElement('metadata') + for (key, value) in meta_items: + item_node = self._meta_item_to_xml(xml_doc, key, value) + container_node.appendChild(item_node) + return container_node + + def _meta_list_to_xml_string(self, metadata_dict): + xml_doc = minidom.Document() + items = metadata_dict['metadata'].items() + container_node = self._meta_list_to_xml(xml_doc, items) + self._add_xmlns(container_node) + return container_node.toprettyxml(indent=' ') + + def index(self, metadata_dict): + return self._meta_list_to_xml_string(metadata_dict) + + def create(self, metadata_dict): + return self._meta_list_to_xml_string(metadata_dict) + + def _meta_item_to_xml_string(self, meta_item_dict): + xml_doc = minidom.Document() + item_key, item_value = meta_item_dict.items()[0] + item_node = self._meta_item_to_xml(xml_doc, item_key, item_value) + self._add_xmlns(item_node) + return item_node.toprettyxml(indent=' ') + + def show(self, meta_item_dict): + return self._meta_item_to_xml_string(meta_item_dict['meta']) + + def update(self, meta_item_dict): + return self._meta_item_to_xml_string(meta_item_dict['meta']) + + +def create_resource(): + serializers = { + 'application/xml': ImageMetadataXMLSerializer(), + } + + return wsgi.Resource(Controller(), serializers=serializers) diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index 34d4c27fc..d43340e10 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -13,88 +13,70 @@ # License for the specific language governing permissions and limitations # under the License. +import os.path + import webob.exc from nova import compute from nova import exception from nova import flags +import nova.image from nova import log from nova import utils from nova.api.openstack import common from nova.api.openstack import faults from nova.api.openstack.views import images as images_view +from nova.api.openstack import wsgi LOG = log.getLogger('nova.api.openstack.images') FLAGS = flags.FLAGS +SUPPORTED_FILTERS = ['name', 'status'] -class Controller(common.OpenstackController): - """Base `wsgi.Controller` for retrieving/displaying images.""" - _serialization_metadata = { - 'application/xml': { - "attributes": { - "image": ["id", "name", "updated", "created", "status", - "serverId", "progress"], - "link": ["rel", "type", "href"], - }, - }, - } +class Controller(object): + """Base controller for retrieving/displaying images.""" def __init__(self, image_service=None, compute_service=None): """Initialize new `ImageController`. :param compute_service: `nova.compute.api:API` :param image_service: `nova.image.service:BaseImageService` - """ - _default_service = utils.import_object(flags.FLAGS.image_service) + """ self._compute_service = compute_service or compute.API() - self._image_service = image_service or _default_service - - def index(self, req): - """Return an index listing of images available to the request. + self._image_service = image_service or \ + nova.image.get_default_image_service() - :param req: `wsgi.Request` object + def _get_filters(self, req): """ - context = req.environ['nova.context'] - images = self._image_service.index(context) - images = common.limited(images, req) - builder = self.get_builder(req).build - return dict(images=[builder(image, detail=False) for image in images]) - - def detail(self, req): - """Return a detailed index listing of images available to the request. + Return a dictionary of query param filters from the request - :param req: `wsgi.Request` object. + :param req: the Request object coming from the wsgi layer + :retval a dict of key/value filters """ - context = req.environ['nova.context'] - images = self._image_service.detail(context) - images = common.limited(images, req) - builder = self.get_builder(req).build - return dict(images=[builder(image, detail=True) for image in images]) + filters = {} + for param in req.str_params: + if param in SUPPORTED_FILTERS or param.startswith('property-'): + filters[param] = req.str_params.get(param) + + return filters def show(self, req, id): """Return detailed information about a specific image. :param req: `wsgi.Request` object - :param id: Image identifier (integer) + :param id: Image identifier """ context = req.environ['nova.context'] try: - image_id = int(id) - except ValueError: + image = self._image_service.show(context, id) + except (exception.NotFound, exception.InvalidImageRef): explanation = _("Image not found.") raise faults.Fault(webob.exc.HTTPNotFound(explanation=explanation)) - try: - image = self._image_service.show(context, image_id) - except exception.NotFound: - explanation = _("Image '%d' not found.") % (image_id) - raise faults.Fault(webob.exc.HTTPNotFound(explanation=explanation)) - return dict(image=self.get_builder(req).build(image, detail=True)) def delete(self, req, id): @@ -103,36 +85,43 @@ class Controller(common.OpenstackController): :param req: `wsgi.Request` object :param id: Image identifier (integer) """ - image_id = id context = req.environ['nova.context'] - self._image_service.delete(context, image_id) + self._image_service.delete(context, id) return webob.exc.HTTPNoContent() - def create(self, req): + def create(self, req, body): """Snapshot a server instance and save the image. :param req: `wsgi.Request` object """ context = req.environ['nova.context'] content_type = req.get_content_type() - image = self._deserialize(req.body, content_type) - if not image: + if not body: raise webob.exc.HTTPBadRequest() try: - server_id = image["image"]["serverId"] - image_name = image["image"]["name"] + server_id = self._server_id_from_req(req, body) + image_name = body["image"]["name"] except KeyError: raise webob.exc.HTTPBadRequest() - image = self._compute_service.snapshot(context, server_id, image_name) + props = self._get_extra_properties(req, body) + + image = self._compute_service.snapshot(context, server_id, + image_name, props) return dict(image=self.get_builder(req).build(image, detail=True)) def get_builder(self, request): """Indicates that you must use a Controller subclass.""" raise NotImplementedError + def _server_id_from_req(self, req, data): + raise NotImplementedError() + + def _get_extra_properties(self, req, data): + return {} + class ControllerV10(Controller): """Version 1.0 specific controller logic.""" @@ -142,6 +131,39 @@ class ControllerV10(Controller): base_url = request.application_url return images_view.ViewBuilderV10(base_url) + def index(self, req): + """Return an index listing of images available to the request. + + :param req: `wsgi.Request` object + + """ + context = req.environ['nova.context'] + filters = self._get_filters(req) + images = self._image_service.index(context, filters) + images = common.limited(images, req) + builder = self.get_builder(req).build + return dict(images=[builder(image, detail=False) for image in images]) + + def detail(self, req): + """Return a detailed index listing of images available to the request. + + :param req: `wsgi.Request` object. + + """ + context = req.environ['nova.context'] + filters = self._get_filters(req) + images = self._image_service.detail(context, filters) + images = common.limited(images, req) + builder = self.get_builder(req).build + return dict(images=[builder(image, detail=True) for image in images]) + + def _server_id_from_req(self, req, data): + try: + return data['image']['serverId'] + except KeyError: + msg = _("Expected serverId attribute on server entity.") + raise webob.exc.HTTPBadRequest(explanation=msg) + class ControllerV11(Controller): """Version 1.1 specific controller logic.""" @@ -151,5 +173,79 @@ class ControllerV11(Controller): base_url = request.application_url return images_view.ViewBuilderV11(base_url) - def get_default_xmlns(self, req): - return common.XML_NS_V11 + def index(self, req): + """Return an index listing of images available to the request. + + :param req: `wsgi.Request` object + + """ + context = req.environ['nova.context'] + filters = self._get_filters(req) + (marker, limit) = common.get_pagination_params(req) + images = self._image_service.index( + context, filters=filters, marker=marker, limit=limit) + builder = self.get_builder(req).build + return dict(images=[builder(image, detail=False) for image in images]) + + def detail(self, req): + """Return a detailed index listing of images available to the request. + + :param req: `wsgi.Request` object. + + """ + context = req.environ['nova.context'] + filters = self._get_filters(req) + (marker, limit) = common.get_pagination_params(req) + images = self._image_service.detail( + context, filters=filters, marker=marker, limit=limit) + builder = self.get_builder(req).build + return dict(images=[builder(image, detail=True) for image in images]) + + def _server_id_from_req(self, req, data): + try: + server_ref = data['image']['serverRef'] + except KeyError: + msg = _("Expected serverRef attribute on server entity.") + raise webob.exc.HTTPBadRequest(explanation=msg) + + head, tail = os.path.split(server_ref) + + if head and head != os.path.join(req.application_url, 'servers'): + msg = _("serverRef must match request url") + raise webob.exc.HTTPBadRequest(explanation=msg) + + return tail + + def _get_extra_properties(self, req, data): + server_ref = data['image']['serverRef'] + if not server_ref.startswith('http'): + server_ref = os.path.join(req.application_url, 'servers', + server_ref) + return {'instance_ref': server_ref} + + +def create_resource(version='1.0'): + controller = { + '1.0': ControllerV10, + '1.1': ControllerV11, + }[version]() + + xmlns = { + '1.0': wsgi.XMLNS_V10, + '1.1': wsgi.XMLNS_V11, + }[version] + + metadata = { + "attributes": { + "image": ["id", "name", "updated", "created", "status", + "serverId", "progress", "serverRef"], + "link": ["rel", "type", "href"], + }, + } + + serializers = { + 'application/xml': wsgi.XMLDictSerializer(xmlns=xmlns, + metadata=metadata), + } + + return wsgi.Resource(controller, serializers=serializers) diff --git a/nova/api/openstack/ips.py b/nova/api/openstack/ips.py index 778e9ba1a..71646b6d3 100644 --- a/nova/api/openstack/ips.py +++ b/nova/api/openstack/ips.py @@ -20,53 +20,59 @@ import time from webob import exc import nova -import nova.api.openstack.views.addresses -from nova.api.openstack import common from nova.api.openstack import faults +import nova.api.openstack.views.addresses +from nova.api.openstack import wsgi -class Controller(common.OpenstackController): +class Controller(object): """The servers addresses API controller for the Openstack API.""" - _serialization_metadata = { - 'application/xml': { - 'list_collections': { - 'public': {'item_name': 'ip', 'item_key': 'addr'}, - 'private': {'item_name': 'ip', 'item_key': 'addr'}, - }, - }, - } - def __init__(self): self.compute_api = nova.compute.API() self.builder = nova.api.openstack.views.addresses.ViewBuilderV10() - def index(self, req, server_id): + def _get_instance(self, req, server_id): try: - instance = self.compute_api.get(req.environ['nova.context'], id) + instance = self.compute_api.get( + req.environ['nova.context'], server_id) except nova.exception.NotFound: return faults.Fault(exc.HTTPNotFound()) + return instance + + def index(self, req, server_id): + instance = self._get_instance(req, server_id) return {'addresses': self.builder.build(instance)} def public(self, req, server_id): - try: - instance = self.compute_api.get(req.environ['nova.context'], id) - except nova.exception.NotFound: - return faults.Fault(exc.HTTPNotFound()) + instance = self._get_instance(req, server_id) return {'public': self.builder.build_public_parts(instance)} def private(self, req, server_id): - try: - instance = self.compute_api.get(req.environ['nova.context'], id) - except nova.exception.NotFound: - return faults.Fault(exc.HTTPNotFound()) + instance = self._get_instance(req, server_id) return {'private': self.builder.build_private_parts(instance)} def show(self, req, server_id, id): return faults.Fault(exc.HTTPNotImplemented()) - def create(self, req, server_id): + def create(self, req, server_id, body): return faults.Fault(exc.HTTPNotImplemented()) def delete(self, req, server_id, id): return faults.Fault(exc.HTTPNotImplemented()) + + +def create_resource(): + metadata = { + 'list_collections': { + 'public': {'item_name': 'ip', 'item_key': 'addr'}, + 'private': {'item_name': 'ip', 'item_key': 'addr'}, + }, + } + + serializers = { + 'application/xml': wsgi.XMLDictSerializer(metadata=metadata, + xmlns=wsgi.XMLNS_V10), + } + + return wsgi.Resource(Controller(), serializers=serializers) diff --git a/nova/api/openstack/limits.py b/nova/api/openstack/limits.py index 47bc238f1..fede96e33 100644 --- a/nova/api/openstack/limits.py +++ b/nova/api/openstack/limits.py @@ -11,7 +11,7 @@ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations -# under the License.import datetime +# under the License. """ Module dedicated functions/classes dealing with rate limiting requests. @@ -30,10 +30,13 @@ from collections import defaultdict from webob.dec import wsgify +from nova import quota +from nova import wsgi as base_wsgi from nova import wsgi from nova.api.openstack import common from nova.api.openstack import faults from nova.api.openstack.views import limits as limits_views +from nova.api.openstack import wsgi # Convenience constants for the limits dictionary passed to Limiter(). @@ -43,28 +46,17 @@ PER_HOUR = 60 * 60 PER_DAY = 60 * 60 * 24 -class LimitsController(common.OpenstackController): +class LimitsController(object): """ Controller for accessing limits in the OpenStack API. """ - _serialization_metadata = { - "application/xml": { - "attributes": { - "limit": ["verb", "URI", "uri", "regex", "value", "unit", - "resetTime", "next-available", "remaining", "name"], - }, - "plurals": { - "rate": "limit", - }, - }, - } - def index(self, req): """ Return all global and rate limit information. """ - abs_limits = {} + context = req.environ['nova.context'] + abs_limits = quota.get_project_quotas(context, context.project_id) rate_limits = req.environ.get("nova.limits", []) builder = self._get_view_builder(req) @@ -84,6 +76,35 @@ class LimitsControllerV11(LimitsController): return limits_views.ViewBuilderV11() +def create_resource(version='1.0'): + controller = { + '1.0': LimitsControllerV10, + '1.1': LimitsControllerV11, + }[version]() + + xmlns = { + '1.0': wsgi.XMLNS_V10, + '1.1': wsgi.XMLNS_V11, + }[version] + + metadata = { + "attributes": { + "limit": ["verb", "URI", "uri", "regex", "value", "unit", + "resetTime", "next-available", "remaining", "name"], + }, + "plurals": { + "rate": "limit", + }, + } + + serializers = { + 'application/xml': wsgi.XMLDictSerializer(xmlns=xmlns, + metadata=metadata), + } + + return wsgi.Resource(controller, serializers=serializers) + + class Limit(object): """ Stores information about a limit for HTTP requets. @@ -195,7 +216,7 @@ DEFAULT_LIMITS = [ ] -class RateLimitingMiddleware(wsgi.Middleware): +class RateLimitingMiddleware(base_wsgi.Middleware): """ Rate-limits requests passing through this middleware. All limit information is stored in memory for this implementation. @@ -209,7 +230,7 @@ class RateLimitingMiddleware(wsgi.Middleware): @param application: WSGI application to wrap @param limits: List of dictionaries describing limits """ - wsgi.Middleware.__init__(self, application) + base_wsgi.Middleware.__init__(self, application) self._limiter = Limiter(limits or DEFAULT_LIMITS) @wsgify(RequestClass=wsgi.Request) diff --git a/nova/api/openstack/notes.txt b/nova/api/openstack/notes.txt index 2330f1002..4e95bffc8 100644 --- a/nova/api/openstack/notes.txt +++ b/nova/api/openstack/notes.txt @@ -7,9 +7,6 @@ image ids. GlanceImageService(ImageService): image ids are URIs. -LocalImageService(ImageService): -image ids are random strings. - OpenstackAPITranslationStore: translates RS server/images/flavor/etc ids into formats required by a given ImageService strategy. diff --git a/nova/api/openstack/ratelimiting/__init__.py b/nova/api/openstack/ratelimiting/__init__.py index 88ffc3246..9ede548c2 100644 --- a/nova/api/openstack/ratelimiting/__init__.py +++ b/nova/api/openstack/ratelimiting/__init__.py @@ -13,7 +13,7 @@ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations -# under the License.import datetime +# under the License. """Rate limiting of arbitrary actions.""" diff --git a/nova/api/openstack/server_metadata.py b/nova/api/openstack/server_metadata.py index fd64ee4fb..8a314de22 100644 --- a/nova/api/openstack/server_metadata.py +++ b/nova/api/openstack/server_metadata.py @@ -18,13 +18,13 @@ from webob import exc from nova import compute -from nova import quota -from nova import wsgi -from nova.api.openstack import common from nova.api.openstack import faults +from nova.api.openstack import wsgi +from nova import exception +from nova import quota -class Controller(common.OpenstackController): +class Controller(object): """ The server metadata API controller for the Openstack API """ def __init__(self): @@ -38,26 +38,40 @@ class Controller(common.OpenstackController): meta_dict[key] = value return dict(metadata=meta_dict) + def _check_body(self, body): + if body == None or body == "": + expl = _('No Request Body') + raise exc.HTTPBadRequest(explanation=expl) + def index(self, req, server_id): """ Returns the list of metadata for a given instance """ context = req.environ['nova.context'] - return self._get_metadata(context, server_id) + try: + return self._get_metadata(context, server_id) + except exception.InstanceNotFound: + msg = _('Server %(server_id)s does not exist') % locals() + raise exc.HTTPNotFound(explanation=msg) - def create(self, req, server_id): + def create(self, req, server_id, body): + self._check_body(body) context = req.environ['nova.context'] - data = self._deserialize(req.body, req.get_content_type()) - metadata = data.get('metadata') + metadata = body.get('metadata') try: self.compute_api.update_or_create_instance_metadata(context, server_id, metadata) + except exception.InstanceNotFound: + msg = _('Server %(server_id)s does not exist') % locals() + raise exc.HTTPNotFound(explanation=msg) + except quota.QuotaError as error: self._handle_quota_error(error) - return req.body - def update(self, req, server_id, id): + return body + + def update(self, req, server_id, id, body): + self._check_body(body) context = req.environ['nova.context'] - body = self._deserialize(req.body, req.get_content_type()) if not id in body: expl = _('Request body and URI mismatch') raise exc.HTTPBadRequest(explanation=expl) @@ -68,27 +82,49 @@ class Controller(common.OpenstackController): self.compute_api.update_or_create_instance_metadata(context, server_id, body) + except exception.InstanceNotFound: + msg = _('Server %(server_id)s does not exist') % locals() + raise exc.HTTPNotFound(explanation=msg) + except quota.QuotaError as error: self._handle_quota_error(error) - return req.body + return body def show(self, req, server_id, id): """ Return a single metadata item """ context = req.environ['nova.context'] - data = self._get_metadata(context, server_id) - if id in data['metadata']: + try: + data = self._get_metadata(context, server_id) + except exception.InstanceNotFound: + msg = _('Server %(server_id)s does not exist') % locals() + raise exc.HTTPNotFound(explanation=msg) + + try: return {id: data['metadata'][id]} - else: - return faults.Fault(exc.HTTPNotFound()) + except KeyError: + msg = _("metadata item %s was not found" % (id)) + raise exc.HTTPNotFound(explanation=msg) def delete(self, req, server_id, id): """ Deletes an existing metadata """ context = req.environ['nova.context'] - self.compute_api.delete_instance_metadata(context, server_id, id) + try: + self.compute_api.delete_instance_metadata(context, server_id, id) + except exception.InstanceNotFound: + msg = _('Server %(server_id)s does not exist') % locals() + raise exc.HTTPNotFound(explanation=msg) def _handle_quota_error(self, error): """Reraise quota errors as api-specific http exceptions.""" if error.code == "MetadataLimitExceeded": raise exc.HTTPBadRequest(explanation=error.message) raise error + + +def create_resource(): + serializers = { + 'application/xml': wsgi.XMLDictSerializer(xmlns=wsgi.XMLNS_V11), + } + + return wsgi.Resource(Controller(), serializers=serializers) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 8f2de2afe..b82a6de19 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -17,22 +17,20 @@ import base64 import traceback from webob import exc -from xml.dom import minidom from nova import compute from nova import exception from nova import flags from nova import log as logging -from nova import quota from nova import utils from nova.api.openstack import common +from nova.api.openstack import create_instance_helper as helper from nova.api.openstack import faults import nova.api.openstack.views.addresses import nova.api.openstack.views.flavors import nova.api.openstack.views.images import nova.api.openstack.views.servers -from nova.auth import manager as auth_manager -from nova.compute import instance_types +from nova.api.openstack import wsgi import nova.api.openstack from nova.scheduler import api as scheduler_api @@ -41,45 +39,28 @@ LOG = logging.getLogger('nova.api.openstack.servers') FLAGS = flags.FLAGS -class Controller(common.OpenstackController): +class Controller(object): """ The Server API controller for the OpenStack API """ - _serialization_metadata = { - "application/xml": { - "attributes": { - "server": ["id", "imageId", "name", "flavorId", "hostId", - "status", "progress", "adminPass", "flavorRef", - "imageRef"], - "link": ["rel", "type", "href"], - }, - "dict_collections": { - "metadata": {"item_name": "meta", "item_key": "key"}, - }, - "list_collections": { - "public": {"item_name": "ip", "item_key": "addr"}, - "private": {"item_name": "ip", "item_key": "addr"}, - }, - }, - } - def __init__(self): self.compute_api = compute.API() - self._image_service = utils.import_object(FLAGS.image_service) - super(Controller, self).__init__() + self.helper = helper.CreateInstanceHelper(self) def index(self, req): """ Returns a list of server names and ids for a given user """ - return self._items(req, is_detail=False) + try: + servers = self._items(req, is_detail=False) + except exception.Invalid as err: + return exc.HTTPBadRequest(explanation=str(err)) + return servers def detail(self, req): """ Returns a list of server details for a given user """ - return self._items(req, is_detail=True) - - def _image_id_from_req_data(self, data): - raise NotImplementedError() - - def _flavor_id_from_req_data(self, data): - raise NotImplementedError() + try: + servers = self._items(req, is_detail=True) + except exception.Invalid as err: + return exc.HTTPBadRequest(explanation=str(err)) + return servers def _get_view_builder(self, req): raise NotImplementedError() @@ -95,7 +76,10 @@ class Controller(common.OpenstackController): builder - the response model builder """ - instance_list = self.compute_api.get_all(req.environ['nova.context']) + reservation_id = req.str_GET.get('reservation_id') + instance_list = self.compute_api.get_all( + req.environ['nova.context'], + reservation_id=reservation_id) limited_list = self._limit_items(instance_list, req) builder = self._get_view_builder(req) servers = [builder.build(inst, is_detail)['server'] @@ -122,157 +106,45 @@ class Controller(common.OpenstackController): return faults.Fault(exc.HTTPNotFound()) return exc.HTTPAccepted() - def create(self, req): + def create(self, req, body): """ Creates a new server for a given user """ - env = self._deserialize_create(req) - if not env: - return faults.Fault(exc.HTTPUnprocessableEntity()) - - context = req.environ['nova.context'] - - password = self._get_server_admin_password(env['server']) - - key_name = None - key_data = None - key_pairs = auth_manager.AuthManager.get_key_pairs(context) - if key_pairs: - key_pair = key_pairs[0] - key_name = key_pair['name'] - key_data = key_pair['public_key'] - - requested_image_id = self._image_id_from_req_data(env) + extra_values = None + result = None try: - image_id = common.get_image_id_from_image_hash(self._image_service, - context, requested_image_id) - except: - msg = _("Can not find requested image") - return faults.Fault(exc.HTTPBadRequest(msg)) - - kernel_id, ramdisk_id = self._get_kernel_ramdisk_from_image( - req, image_id) - - personality = env['server'].get('personality') - injected_files = [] - if personality: - injected_files = self._get_injected_files(personality) + extra_values, result = self.helper.create_instance( + req, body, self.compute_api.create) + except faults.Fault, f: + return f - flavor_id = self._flavor_id_from_req_data(env) + instances = result - if not 'name' in env['server']: - msg = _("Server name is not defined") - return exc.HTTPBadRequest(msg) - - name = env['server']['name'] - self._validate_server_name(name) - name = name.strip() - - try: - inst_type = \ - instance_types.get_instance_type_by_flavor_id(flavor_id) - (inst,) = self.compute_api.create( - context, - inst_type, - image_id, - kernel_id=kernel_id, - ramdisk_id=ramdisk_id, - display_name=name, - display_description=name, - key_name=key_name, - key_data=key_data, - metadata=env['server'].get('metadata', {}), - injected_files=injected_files) - except quota.QuotaError as error: - self._handle_quota_error(error) - - inst['instance_type'] = inst_type - inst['image_id'] = requested_image_id + (inst, ) = instances + for key in ['instance_type', 'image_ref']: + inst[key] = extra_values[key] builder = self._get_view_builder(req) server = builder.build(inst, is_detail=True) - server['server']['adminPass'] = password - self.compute_api.set_admin_password(context, server['server']['id'], - password) + server['server']['adminPass'] = extra_values['password'] return server - def _deserialize_create(self, request): - """ - Deserialize a create request - - Overrides normal behavior in the case of xml content - """ - if request.content_type == "application/xml": - deserializer = ServerCreateRequestXMLDeserializer() - return deserializer.deserialize(request.body) - else: - return self._deserialize(request.body, request.get_content_type()) - - def _get_injected_files(self, personality): - """ - Create a list of injected files from the personality attribute - - At this time, injected_files must be formatted as a list of - (file_path, file_content) pairs for compatibility with the - underlying compute service. - """ - injected_files = [] - - for item in personality: - try: - path = item['path'] - contents = item['contents'] - except KeyError as key: - expl = _('Bad personality format: missing %s') % key - raise exc.HTTPBadRequest(explanation=expl) - except TypeError: - expl = _('Bad personality format') - raise exc.HTTPBadRequest(explanation=expl) - try: - contents = base64.b64decode(contents) - except TypeError: - expl = _('Personality content for %s cannot be decoded') % path - raise exc.HTTPBadRequest(explanation=expl) - injected_files.append((path, contents)) - return injected_files - - def _handle_quota_error(self, error): - """ - Reraise quota errors as api-specific http exceptions - """ - if error.code == "OnsetFileLimitExceeded": - expl = _("Personality file limit exceeded") - raise exc.HTTPBadRequest(explanation=expl) - if error.code == "OnsetFilePathLimitExceeded": - expl = _("Personality file path too long") - raise exc.HTTPBadRequest(explanation=expl) - if error.code == "OnsetFileContentLimitExceeded": - expl = _("Personality file content too long") - raise exc.HTTPBadRequest(explanation=expl) - # if the original error is okay, just reraise it - raise error - - def _get_server_admin_password(self, server): - """ Determine the admin password for a server on creation """ - return utils.generate_password(16) - @scheduler_api.redirect_handler - def update(self, req, id): + def update(self, req, id, body): """ Updates the server name or password """ if len(req.body) == 0: raise exc.HTTPUnprocessableEntity() - inst_dict = self._deserialize(req.body, req.get_content_type()) - if not inst_dict: + if not body: return faults.Fault(exc.HTTPUnprocessableEntity()) ctxt = req.environ['nova.context'] update_dict = {} - if 'name' in inst_dict['server']: - name = inst_dict['server']['name'] - self._validate_server_name(name) + if 'name' in body['server']: + name = body['server']['name'] + self.helper._validate_server_name(name) update_dict['display_name'] = name.strip() - self._parse_update(ctxt, id, inst_dict, update_dict) + self._parse_update(ctxt, id, body, update_dict) try: self.compute_api.update(ctxt, id, **update_dict) @@ -281,20 +153,11 @@ class Controller(common.OpenstackController): return exc.HTTPNoContent() - def _validate_server_name(self, value): - if not isinstance(value, basestring): - msg = _("Server name is not a string or unicode") - raise exc.HTTPBadRequest(msg) - - if value.strip() == '': - msg = _("Server name is an empty string") - raise exc.HTTPBadRequest(msg) - def _parse_update(self, context, id, inst_dict, update_dict): pass @scheduler_api.redirect_handler - def action(self, req, id): + def action(self, req, id, body): """Multi-purpose method used to reboot, rebuild, or resize a server""" @@ -307,10 +170,9 @@ class Controller(common.OpenstackController): 'rebuild': self._action_rebuild, } - input_dict = self._deserialize(req.body, req.get_content_type()) for key in actions.keys(): - if key in input_dict: - return actions[key](input_dict, req, id) + if key in body: + return actions[key](body, req, id) return faults.Fault(exc.HTTPNotImplemented()) def _action_change_password(self, input_dict, req, id): @@ -333,19 +195,7 @@ class Controller(common.OpenstackController): return exc.HTTPAccepted() def _action_resize(self, input_dict, req, id): - """ Resizes a given instance to the flavor size requested """ - try: - if 'resize' in input_dict and 'flavorId' in input_dict['resize']: - flavor_id = input_dict['resize']['flavorId'] - self.compute_api.resize(req.environ['nova.context'], id, - flavor_id) - else: - LOG.exception(_("Missing arguments for resize")) - return faults.Fault(exc.HTTPUnprocessableEntity()) - except Exception, e: - LOG.exception(_("Error in resize %s"), e) - return faults.Fault(exc.HTTPBadRequest()) - return exc.HTTPAccepted() + return exc.HTTPNotImplemented() def _action_reboot(self, input_dict, req, id): if 'reboot' in input_dict and 'type' in input_dict['reboot']: @@ -410,7 +260,7 @@ class Controller(common.OpenstackController): return exc.HTTPAccepted() @scheduler_api.redirect_handler - def reset_network(self, req, id): + def reset_network(self, req, id, body): """ Reset networking on an instance (admin only). @@ -425,7 +275,7 @@ class Controller(common.OpenstackController): return exc.HTTPAccepted() @scheduler_api.redirect_handler - def inject_network_info(self, req, id): + def inject_network_info(self, req, id, body): """ Inject network info for an instance (admin only). @@ -440,7 +290,7 @@ class Controller(common.OpenstackController): return exc.HTTPAccepted() @scheduler_api.redirect_handler - def pause(self, req, id): + def pause(self, req, id, body): """ Permit Admins to Pause the server. """ ctxt = req.environ['nova.context'] try: @@ -452,7 +302,7 @@ class Controller(common.OpenstackController): return exc.HTTPAccepted() @scheduler_api.redirect_handler - def unpause(self, req, id): + def unpause(self, req, id, body): """ Permit Admins to Unpause the server. """ ctxt = req.environ['nova.context'] try: @@ -464,7 +314,7 @@ class Controller(common.OpenstackController): return exc.HTTPAccepted() @scheduler_api.redirect_handler - def suspend(self, req, id): + def suspend(self, req, id, body): """permit admins to suspend the server""" context = req.environ['nova.context'] try: @@ -476,7 +326,7 @@ class Controller(common.OpenstackController): return exc.HTTPAccepted() @scheduler_api.redirect_handler - def resume(self, req, id): + def resume(self, req, id, body): """permit admins to resume the server from suspend""" context = req.environ['nova.context'] try: @@ -551,48 +401,10 @@ class Controller(common.OpenstackController): error=item.error)) return dict(actions=actions) - def _get_kernel_ramdisk_from_image(self, req, image_id): - """Fetch an image from the ImageService, then if present, return the - associated kernel and ramdisk image IDs. - """ - context = req.environ['nova.context'] - image_meta = self._image_service.show(context, image_id) - # NOTE(sirp): extracted to a separate method to aid unit-testing, the - # new method doesn't need a request obj or an ImageService stub - kernel_id, ramdisk_id = self._do_get_kernel_ramdisk_from_image( - image_meta) - return kernel_id, ramdisk_id - - @staticmethod - def _do_get_kernel_ramdisk_from_image(image_meta): - """Given an ImageService image_meta, return kernel and ramdisk image - ids if present. - - This is only valid for `ami` style images. - """ - image_id = image_meta['id'] - if image_meta['status'] != 'active': - raise exception.ImageUnacceptable(image_id=image_id, - reason=_("status is not active")) - - if image_meta.get('container_format') != 'ami': - return None, None - - try: - kernel_id = image_meta['properties']['kernel_id'] - except KeyError: - raise exception.KernelNotFoundForImage(image_id=image_id) - - try: - ramdisk_id = image_meta['properties']['ramdisk_id'] - except KeyError: - raise exception.RamdiskNotFoundForImage(image_id=image_id) - - return kernel_id, ramdisk_id - class ControllerV10(Controller): - def _image_id_from_req_data(self, data): + + def _image_ref_from_req_data(self, data): return data['server']['imageId'] def _flavor_id_from_req_data(self, data): @@ -608,8 +420,23 @@ class ControllerV10(Controller): def _parse_update(self, context, server_id, inst_dict, update_dict): if 'adminPass' in inst_dict['server']: - update_dict['admin_pass'] = inst_dict['server']['adminPass'] - self.compute_api.set_admin_password(context, server_id) + self.compute_api.set_admin_password(context, server_id, + inst_dict['server']['adminPass']) + + def _action_resize(self, input_dict, req, id): + """ Resizes a given instance to the flavor size requested """ + try: + if 'resize' in input_dict and 'flavorId' in input_dict['resize']: + flavor_id = input_dict['resize']['flavorId'] + self.compute_api.resize(req.environ['nova.context'], id, + flavor_id) + else: + LOG.exception(_("Missing 'flavorId' argument for resize")) + return faults.Fault(exc.HTTPUnprocessableEntity()) + except Exception, e: + LOG.exception(_("Error in resize %s"), e) + return faults.Fault(exc.HTTPBadRequest()) + return exc.HTTPAccepted() def _action_rebuild(self, info, request, instance_id): context = request.environ['nova.context'] @@ -633,11 +460,14 @@ class ControllerV10(Controller): response.empty_body = True return response + def _get_server_admin_password(self, server): + """ Determine the admin password for a server on creation """ + return self.helper._get_server_admin_password_old_style(server) + class ControllerV11(Controller): - def _image_id_from_req_data(self, data): - href = data['server']['imageRef'] - return common.get_id_from_href(href) + def _image_ref_from_req_data(self, data): + return data['server']['imageRef'] def _flavor_id_from_req_data(self, data): href = data['server']['flavorRef'] @@ -658,11 +488,11 @@ class ControllerV11(Controller): if (not 'changePassword' in input_dict or not 'adminPass' in input_dict['changePassword']): msg = _("No adminPass was specified") - return exc.HTTPBadRequest(msg) + return exc.HTTPBadRequest(explanation=msg) password = input_dict['changePassword']['adminPass'] if not isinstance(password, basestring) or password == '': msg = _("Invalid adminPass") - return exc.HTTPBadRequest(msg) + return exc.HTTPBadRequest(explanation=msg) self.compute_api.set_admin_password(context, id, password) return exc.HTTPAccepted() @@ -696,27 +526,44 @@ class ControllerV11(Controller): LOG.info(msg) raise faults.Fault(exc.HTTPBadRequest(explanation=msg)) + def _action_resize(self, input_dict, req, id): + """ Resizes a given instance to the flavor size requested """ + try: + if 'resize' in input_dict and 'flavorRef' in input_dict['resize']: + flavor_ref = input_dict['resize']['flavorRef'] + flavor_id = common.get_id_from_href(flavor_ref) + self.compute_api.resize(req.environ['nova.context'], id, + flavor_id) + else: + LOG.exception(_("Missing 'flavorRef' argument for resize")) + return faults.Fault(exc.HTTPUnprocessableEntity()) + except Exception, e: + LOG.exception(_("Error in resize %s"), e) + return faults.Fault(exc.HTTPBadRequest()) + return exc.HTTPAccepted() + def _action_rebuild(self, info, request, instance_id): context = request.environ['nova.context'] instance_id = int(instance_id) try: - image_ref = info["rebuild"]["imageRef"] + image_href = info["rebuild"]["imageRef"] except (KeyError, TypeError): msg = _("Could not parse imageRef from request.") LOG.debug(msg) return faults.Fault(exc.HTTPBadRequest(explanation=msg)) - image_id = common.get_id_from_href(image_ref) personalities = info["rebuild"].get("personality", []) - metadata = info["rebuild"].get("metadata", {}) + metadata = info["rebuild"].get("metadata") + name = info["rebuild"].get("name") - self._validate_metadata(metadata) + if metadata: + self._validate_metadata(metadata) self._decode_personalities(personalities) try: - self.compute_api.rebuild(context, instance_id, image_id, metadata, - personalities) + self.compute_api.rebuild(context, instance_id, image_href, name, + metadata, personalities) except exception.BuildInProgress: msg = _("Instance %d is currently being rebuilt.") % instance_id LOG.debug(msg) @@ -726,92 +573,49 @@ class ControllerV11(Controller): response.empty_body = True return response + def get_default_xmlns(self, req): + return common.XML_NS_V11 + def _get_server_admin_password(self, server): """ Determine the admin password for a server on creation """ - password = server.get('adminPass') - if password is None: - return utils.generate_password(16) - if not isinstance(password, basestring) or password == '': - msg = _("Invalid adminPass") - raise exc.HTTPBadRequest(msg) - return password + return self.helper._get_server_admin_password_new_style(server) - def get_default_xmlns(self, req): - return common.XML_NS_V11 +def create_resource(version='1.0'): + controller = { + '1.0': ControllerV10, + '1.1': ControllerV11, + }[version]() -class ServerCreateRequestXMLDeserializer(object): - """ - Deserializer to handle xml-formatted server create requests. - - Handles standard server attributes as well as optional metadata - and personality attributes - """ - - def deserialize(self, string): - """Deserialize an xml-formatted server create request""" - dom = minidom.parseString(string) - server = self._extract_server(dom) - return {'server': server} - - def _extract_server(self, node): - """Marshal the server attribute of a parsed request""" - server = {} - server_node = self._find_first_child_named(node, 'server') - for attr in ["name", "imageId", "flavorId", "imageRef", "flavorRef"]: - if server_node.getAttribute(attr): - server[attr] = server_node.getAttribute(attr) - metadata = self._extract_metadata(server_node) - if metadata is not None: - server["metadata"] = metadata - personality = self._extract_personality(server_node) - if personality is not None: - server["personality"] = personality - return server + metadata = { + "attributes": { + "server": ["id", "imageId", "name", "flavorId", "hostId", + "status", "progress", "adminPass", "flavorRef", + "imageRef"], + "link": ["rel", "type", "href"], + }, + "dict_collections": { + "metadata": {"item_name": "meta", "item_key": "key"}, + }, + "list_collections": { + "public": {"item_name": "ip", "item_key": "addr"}, + "private": {"item_name": "ip", "item_key": "addr"}, + }, + } + + xmlns = { + '1.0': wsgi.XMLNS_V10, + '1.1': wsgi.XMLNS_V11, + }[version] + + serializers = { + 'application/xml': wsgi.XMLDictSerializer(metadata=metadata, + xmlns=xmlns), + } + + deserializers = { + 'application/xml': helper.ServerXMLDeserializer(), + } - def _extract_metadata(self, server_node): - """Marshal the metadata attribute of a parsed request""" - metadata_node = self._find_first_child_named(server_node, "metadata") - if metadata_node is None: - return None - metadata = {} - for meta_node in self._find_children_named(metadata_node, "meta"): - key = meta_node.getAttribute("key") - metadata[key] = self._extract_text(meta_node) - return metadata - - def _extract_personality(self, server_node): - """Marshal the personality attribute of a parsed request""" - personality_node = \ - self._find_first_child_named(server_node, "personality") - if personality_node is None: - return None - personality = [] - for file_node in self._find_children_named(personality_node, "file"): - item = {} - if file_node.hasAttribute("path"): - item["path"] = file_node.getAttribute("path") - item["contents"] = self._extract_text(file_node) - personality.append(item) - return personality - - def _find_first_child_named(self, parent, name): - """Search a nodes children for the first child with a given name""" - for node in parent.childNodes: - if node.nodeName == name: - return node - return None - - def _find_children_named(self, parent, name): - """Return all of a nodes children who have the given name""" - for node in parent.childNodes: - if node.nodeName == name: - yield node - - def _extract_text(self, node): - """Get the text field contained by the given node""" - if len(node.childNodes) == 1: - child = node.childNodes[0] - if child.nodeType == child.TEXT_NODE: - return child.nodeValue - return "" + return wsgi.Resource(controller, serializers=serializers, + deserializers=deserializers) diff --git a/nova/api/openstack/shared_ip_groups.py b/nova/api/openstack/shared_ip_groups.py index 996db3648..4f11f8dfb 100644 --- a/nova/api/openstack/shared_ip_groups.py +++ b/nova/api/openstack/shared_ip_groups.py @@ -17,29 +17,13 @@ from webob import exc -from nova.api.openstack import common from nova.api.openstack import faults +from nova.api.openstack import wsgi -def _translate_keys(inst): - """ Coerces a shared IP group instance into proper dictionary format """ - return dict(sharedIpGroup=inst) - - -def _translate_detail_keys(inst): - """ Coerces a shared IP group instance into proper dictionary format with - correctly mapped attributes """ - return dict(sharedIpGroups=inst) - - -class Controller(common.OpenstackController): +class Controller(object): """ The Shared IP Groups Controller for the Openstack API """ - _serialization_metadata = { - 'application/xml': { - 'attributes': { - 'sharedIpGroup': []}}} - def index(self, req): """ Returns a list of Shared IP Groups for the user """ raise faults.Fault(exc.HTTPNotImplemented()) @@ -48,7 +32,7 @@ class Controller(common.OpenstackController): """ Shows in-depth information on a specific Shared IP Group """ raise faults.Fault(exc.HTTPNotImplemented()) - def update(self, req, id): + def update(self, req, id, body): """ You can't update a Shared IP Group """ raise faults.Fault(exc.HTTPNotImplemented()) @@ -60,6 +44,10 @@ class Controller(common.OpenstackController): """ Returns a complete list of Shared IP Groups """ raise faults.Fault(exc.HTTPNotImplemented()) - def create(self, req): + def create(self, req, body): """ Creates a new Shared IP group """ raise faults.Fault(exc.HTTPNotImplemented()) + + +def create_resource(): + return wsgi.Resource(Controller()) diff --git a/nova/api/openstack/users.py b/nova/api/openstack/users.py index 7ae4c3232..50975fc1f 100644 --- a/nova/api/openstack/users.py +++ b/nova/api/openstack/users.py @@ -20,8 +20,10 @@ from nova import flags from nova import log as logging from nova.api.openstack import common from nova.api.openstack import faults +from nova.api.openstack import wsgi from nova.auth import manager + FLAGS = flags.FLAGS LOG = logging.getLogger('nova.api.openstack') @@ -34,12 +36,7 @@ def _translate_keys(user): admin=user.admin) -class Controller(common.OpenstackController): - - _serialization_metadata = { - 'application/xml': { - "attributes": { - "user": ["id", "name", "access", "secret", "admin"]}}} +class Controller(object): def __init__(self): self.manager = manager.AuthManager() @@ -81,23 +78,35 @@ class Controller(common.OpenstackController): self.manager.delete_user(id) return {} - def create(self, req): + def create(self, req, body): self._check_admin(req.environ['nova.context']) - env = self._deserialize(req.body, req.get_content_type()) - is_admin = env['user'].get('admin') in ('T', 'True', True) - name = env['user'].get('name') - access = env['user'].get('access') - secret = env['user'].get('secret') + is_admin = body['user'].get('admin') in ('T', 'True', True) + name = body['user'].get('name') + access = body['user'].get('access') + secret = body['user'].get('secret') user = self.manager.create_user(name, access, secret, is_admin) return dict(user=_translate_keys(user)) - def update(self, req, id): + def update(self, req, id, body): self._check_admin(req.environ['nova.context']) - env = self._deserialize(req.body, req.get_content_type()) - is_admin = env['user'].get('admin') + is_admin = body['user'].get('admin') if is_admin is not None: is_admin = is_admin in ('T', 'True', True) - access = env['user'].get('access') - secret = env['user'].get('secret') + access = body['user'].get('access') + secret = body['user'].get('secret') self.manager.modify_user(id, access, secret, is_admin) return dict(user=_translate_keys(self.manager.get_user(id))) + + +def create_resource(): + metadata = { + "attributes": { + "user": ["id", "name", "access", "secret", "admin"], + }, + } + + serializers = { + 'application/xml': wsgi.XMLDictSerializer(metadata=metadata), + } + + return wsgi.Resource(Controller(), serializers=serializers) diff --git a/nova/api/openstack/versions.py b/nova/api/openstack/versions.py index 3f9d91934..4c682302f 100644 --- a/nova/api/openstack/versions.py +++ b/nova/api/openstack/versions.py @@ -18,13 +18,26 @@ import webob import webob.dec -from nova import wsgi import nova.api.openstack.views.versions +from nova.api.openstack import wsgi -class Versions(wsgi.Application): - @webob.dec.wsgify(RequestClass=wsgi.Request) - def __call__(self, req): +class Versions(wsgi.Resource): + def __init__(self): + metadata = { + "attributes": { + "version": ["status", "id"], + "link": ["rel", "href"], + } + } + + serializers = { + 'application/xml': wsgi.XMLDictSerializer(metadata=metadata), + } + + wsgi.Resource.__init__(self, None, serializers=serializers) + + def dispatch(self, request, *args): """Respond to a request for all OpenStack API versions.""" version_objs = [ { @@ -37,24 +50,6 @@ class Versions(wsgi.Application): }, ] - builder = nova.api.openstack.views.versions.get_view_builder(req) + builder = nova.api.openstack.views.versions.get_view_builder(request) versions = [builder.build(version) for version in version_objs] - response = dict(versions=versions) - - metadata = { - "application/xml": { - "attributes": { - "version": ["status", "id"], - "link": ["rel", "href"], - } - } - } - - content_type = req.best_match_content_type() - body = wsgi.Serializer(metadata).serialize(response, content_type) - - response = webob.Response() - response.content_type = content_type - response.body = body - - return response + return dict(versions=versions) diff --git a/nova/api/openstack/views/images.py b/nova/api/openstack/views/images.py index 2773c9c13..d6a054102 100644 --- a/nova/api/openstack/views/images.py +++ b/nova/api/openstack/views/images.py @@ -46,13 +46,9 @@ class ViewBuilder(object): except KeyError: image['status'] = image['status'].upper() - def _build_server(self, image, instance_id): + def _build_server(self, image, image_obj): """Indicates that you must use a ViewBuilder subclass.""" - raise NotImplementedError - - def generate_server_ref(self, server_id): - """Return an href string pointing to this server.""" - return os.path.join(self._url, "servers", str(server_id)) + raise NotImplementedError() def generate_href(self, image_id): """Return an href string pointing to this object.""" @@ -60,8 +56,6 @@ class ViewBuilder(object): def build(self, image_obj, detail=False): """Return a standardized image structure for display by the API.""" - properties = image_obj.get("properties", {}) - self._format_dates(image_obj) if "status" in image_obj: @@ -72,11 +66,7 @@ class ViewBuilder(object): "name": image_obj.get("name"), } - if "instance_id" in properties: - try: - self._build_server(image, int(properties["instance_id"])) - except ValueError: - pass + self._build_server(image, image_obj) if detail: image.update({ @@ -94,15 +84,21 @@ class ViewBuilder(object): class ViewBuilderV10(ViewBuilder): """OpenStack API v1.0 Image Builder""" - def _build_server(self, image, instance_id): - image["serverId"] = instance_id + def _build_server(self, image, image_obj): + try: + image['serverId'] = int(image_obj['properties']['instance_id']) + except (KeyError, ValueError): + pass class ViewBuilderV11(ViewBuilder): """OpenStack API v1.1 Image Builder""" - def _build_server(self, image, instance_id): - image["serverRef"] = self.generate_server_ref(instance_id) + def _build_server(self, image, image_obj): + try: + image['serverRef'] = image_obj['properties']['instance_ref'] + except KeyError: + return def build(self, image_obj, detail=False): """Return a standardized image structure for display by the API.""" diff --git a/nova/api/openstack/views/limits.py b/nova/api/openstack/views/limits.py index 22d1c260d..934b4921a 100644 --- a/nova/api/openstack/views/limits.py +++ b/nova/api/openstack/views/limits.py @@ -29,9 +29,6 @@ class ViewBuilder(object): def _build_rate_limit(self, rate_limit): raise NotImplementedError() - def _build_absolute_limits(self, absolute_limit): - raise NotImplementedError() - def build(self, rate_limits, absolute_limits): rate_limits = self._build_rate_limits(rate_limits) absolute_limits = self._build_absolute_limits(absolute_limits) @@ -45,6 +42,28 @@ class ViewBuilder(object): return output + def _build_absolute_limits(self, absolute_limits): + """Builder for absolute limits + + absolute_limits should be given as a dict of limits. + For example: {"ram": 512, "gigabytes": 1024}. + + """ + limit_names = { + "ram": ["maxTotalRAMSize"], + "instances": ["maxTotalInstances"], + "cores": ["maxTotalCores"], + "metadata_items": ["maxServerMeta", "maxImageMeta"], + "injected_files": ["maxPersonality"], + "injected_file_content_bytes": ["maxPersonalitySize"], + } + limits = {} + for name, value in absolute_limits.iteritems(): + if name in limit_names and value is not None: + for name in limit_names[name]: + limits[name] = value + return limits + class ViewBuilderV10(ViewBuilder): """Openstack API v1.0 limits view builder.""" @@ -63,9 +82,6 @@ class ViewBuilderV10(ViewBuilder): "resetTime": rate_limit["resetTime"], } - def _build_absolute_limits(self, absolute_limit): - return {} - class ViewBuilderV11(ViewBuilder): """Openstack API v1.1 limits view builder.""" @@ -79,7 +95,7 @@ class ViewBuilderV11(ViewBuilder): # check for existing key for limit in limits: if limit["uri"] == rate_limit["URI"] and \ - limit["regex"] == limit["regex"]: + limit["regex"] == rate_limit["regex"]: _rate_limit_key = limit break @@ -104,6 +120,3 @@ class ViewBuilderV11(ViewBuilder): "unit": rate_limit["unit"], "next-available": rate_limit["resetTime"], } - - def _build_absolute_limits(self, absolute_limit): - return {} diff --git a/nova/api/openstack/views/servers.py b/nova/api/openstack/views/servers.py index 0be468edc..cbfa5aae7 100644 --- a/nova/api/openstack/views/servers.py +++ b/nova/api/openstack/views/servers.py @@ -18,6 +18,7 @@ import hashlib import os +from nova import exception from nova.compute import power_state import nova.compute import nova.context @@ -41,12 +42,15 @@ class ViewBuilder(object): def build(self, inst, is_detail): """Return a dict that represenst a server.""" - if is_detail: - server = self._build_detail(inst) + if inst.get('_is_precooked', False): + server = dict(server=inst) else: - server = self._build_simple(inst) + if is_detail: + server = self._build_detail(inst) + else: + server = self._build_simple(inst) - self._build_extra(server, inst) + self._build_extra(server, inst) return server @@ -71,13 +75,14 @@ class ViewBuilder(object): } inst_dict = { - 'id': int(inst['id']), + 'id': inst['id'], 'name': inst['display_name'], 'addresses': self.addresses_builder.build(inst), 'status': power_mapping[inst.get('state')]} ctxt = nova.context.get_admin_context() compute_api = nova.compute.API() + if compute_api.has_finished_migration(ctxt, inst['id']): inst_dict['status'] = 'RESIZE-CONFIRM' @@ -94,6 +99,7 @@ class ViewBuilder(object): self._build_image(inst_dict, inst) self._build_flavor(inst_dict, inst) + inst_dict['uuid'] = inst['uuid'] return dict(server=inst_dict) def _build_image(self, response, inst): @@ -112,8 +118,11 @@ class ViewBuilderV10(ViewBuilder): """Model an Openstack API V1.0 server response.""" def _build_image(self, response, inst): - if 'image_id' in dict(inst): - response['imageId'] = inst['image_id'] + if 'image_ref' in dict(inst): + image_ref = inst['image_ref'] + if str(image_ref).startswith('http'): + raise exception.ListingImageRefsNotSupported() + response['imageId'] = int(image_ref) def _build_flavor(self, response, inst): if 'instance_type' in dict(inst): @@ -130,9 +139,11 @@ class ViewBuilderV11(ViewBuilder): self.base_url = base_url def _build_image(self, response, inst): - if "image_id" in dict(inst): - image_id = inst.get("image_id") - response["imageRef"] = self.image_builder.generate_href(image_id) + if 'image_ref' in dict(inst): + image_href = inst['image_ref'] + if str(image_href).isdigit(): + image_href = int(image_href) + response['imageRef'] = image_href def _build_flavor(self, response, inst): if "instance_type" in dict(inst): diff --git a/nova/api/openstack/wsgi.py b/nova/api/openstack/wsgi.py new file mode 100644 index 000000000..5d24b4cca --- /dev/null +++ b/nova/api/openstack/wsgi.py @@ -0,0 +1,397 @@ + +import json +import webob +from xml.dom import minidom +from xml.parsers import expat + +import faults +from nova import exception +from nova import log as logging +from nova import utils +from nova import wsgi + + +XMLNS_V10 = 'http://docs.rackspacecloud.com/servers/api/v1.0' +XMLNS_V11 = 'http://docs.openstack.org/compute/api/v1.1' + +LOG = logging.getLogger('nova.api.openstack.wsgi') + + +class Request(webob.Request): + """Add some Openstack API-specific logic to the base webob.Request.""" + + def best_match_content_type(self): + """Determine the requested response content-type. + + Based on the query extension then the Accept header. + + """ + supported = ('application/json', 'application/xml') + + parts = self.path.rsplit('.', 1) + if len(parts) > 1: + ctype = 'application/{0}'.format(parts[1]) + if ctype in supported: + return ctype + + bm = self.accept.best_match(supported) + + # default to application/json if we don't find a preference + return bm or 'application/json' + + def get_content_type(self): + """Determine content type of the request body. + + Does not do any body introspection, only checks header + + """ + if not "Content-Type" in self.headers: + raise exception.InvalidContentType(content_type=None) + + allowed_types = ("application/xml", "application/json") + content_type = self.content_type + + if content_type not in allowed_types: + raise exception.InvalidContentType(content_type=content_type) + else: + return content_type + + +class TextDeserializer(object): + """Custom request body deserialization based on controller action name.""" + + def deserialize(self, datastring, action='default'): + """Find local deserialization method and parse request body.""" + action_method = getattr(self, str(action), self.default) + return action_method(datastring) + + def default(self, datastring): + """Default deserialization code should live here""" + raise NotImplementedError() + + +class JSONDeserializer(TextDeserializer): + + def default(self, datastring): + try: + return utils.loads(datastring) + except ValueError: + raise exception.MalformedRequestBody( + reason=_("malformed JSON in request body")) + + +class XMLDeserializer(TextDeserializer): + + def __init__(self, metadata=None): + """ + :param metadata: information needed to deserialize xml into + a dictionary. + """ + super(XMLDeserializer, self).__init__() + self.metadata = metadata or {} + + def default(self, datastring): + plurals = set(self.metadata.get('plurals', {})) + + try: + node = minidom.parseString(datastring).childNodes[0] + return {node.nodeName: self._from_xml_node(node, plurals)} + except expat.ExpatError: + raise exception.MalformedRequestBody( + reason=_("malformed XML in request body")) + + def _from_xml_node(self, node, listnames): + """Convert a minidom node to a simple Python type. + + :param listnames: list of XML node names whose subnodes should + be considered list items. + + """ + if len(node.childNodes) == 1 and node.childNodes[0].nodeType == 3: + return node.childNodes[0].nodeValue + elif node.nodeName in listnames: + return [self._from_xml_node(n, listnames) for n in node.childNodes] + else: + result = dict() + for attr in node.attributes.keys(): + result[attr] = node.attributes[attr].nodeValue + for child in node.childNodes: + if child.nodeType != node.TEXT_NODE: + result[child.nodeName] = self._from_xml_node(child, + listnames) + return result + + +class RequestDeserializer(object): + """Break up a Request object into more useful pieces.""" + + def __init__(self, deserializers=None): + """ + :param deserializers: dictionary of content-type-specific deserializers + + """ + self.deserializers = { + 'application/xml': XMLDeserializer(), + 'application/json': JSONDeserializer(), + } + + self.deserializers.update(deserializers or {}) + + def deserialize(self, request): + """Extract necessary pieces of the request. + + :param request: Request object + :returns tuple of expected controller action name, dictionary of + keyword arguments to pass to the controller, the expected + content type of the response + + """ + action_args = self.get_action_args(request.environ) + action = action_args.pop('action', None) + + if request.method.lower() in ('post', 'put'): + if len(request.body) == 0: + action_args['body'] = None + else: + content_type = request.get_content_type() + deserializer = self.get_deserializer(content_type) + + try: + body = deserializer.deserialize(request.body, action) + action_args['body'] = body + except exception.InvalidContentType: + action_args['body'] = None + + accept = self.get_expected_content_type(request) + + return (action, action_args, accept) + + def get_deserializer(self, content_type): + try: + return self.deserializers[content_type] + except (KeyError, TypeError): + raise exception.InvalidContentType(content_type=content_type) + + def get_expected_content_type(self, request): + return request.best_match_content_type() + + def get_action_args(self, request_environment): + """Parse dictionary created by routes library.""" + try: + args = request_environment['wsgiorg.routing_args'][1].copy() + except Exception: + return {} + + try: + del args['controller'] + except KeyError: + pass + + try: + del args['format'] + except KeyError: + pass + + return args + + +class DictSerializer(object): + """Custom response body serialization based on controller action name.""" + + def serialize(self, data, action='default'): + """Find local serialization method and encode response body.""" + action_method = getattr(self, str(action), self.default) + return action_method(data) + + def default(self, data): + """Default serialization code should live here""" + raise NotImplementedError() + + +class JSONDictSerializer(DictSerializer): + + def default(self, data): + return utils.dumps(data) + + +class XMLDictSerializer(DictSerializer): + + def __init__(self, metadata=None, xmlns=None): + """ + :param metadata: information needed to deserialize xml into + a dictionary. + :param xmlns: XML namespace to include with serialized xml + """ + super(XMLDictSerializer, self).__init__() + self.metadata = metadata or {} + self.xmlns = xmlns + + def default(self, data): + # We expect data to contain a single key which is the XML root. + root_key = data.keys()[0] + doc = minidom.Document() + node = self._to_xml_node(doc, self.metadata, root_key, data[root_key]) + + self._add_xmlns(node) + + return node.toprettyxml(indent=' ', encoding='utf-8') + + def _add_xmlns(self, node): + if self.xmlns is not None: + node.setAttribute('xmlns', self.xmlns) + + def _to_xml_node(self, doc, metadata, nodename, data): + """Recursive method to convert data members to XML nodes.""" + result = doc.createElement(nodename) + + # Set the xml namespace if one is specified + # TODO(justinsb): We could also use prefixes on the keys + xmlns = metadata.get('xmlns', None) + if xmlns: + result.setAttribute('xmlns', xmlns) + + #TODO(bcwaldon): accomplish this without a type-check + if type(data) is list: + collections = metadata.get('list_collections', {}) + if nodename in collections: + metadata = collections[nodename] + for item in data: + node = doc.createElement(metadata['item_name']) + node.setAttribute(metadata['item_key'], str(item)) + result.appendChild(node) + return result + singular = metadata.get('plurals', {}).get(nodename, None) + if singular is None: + if nodename.endswith('s'): + singular = nodename[:-1] + else: + singular = 'item' + for item in data: + node = self._to_xml_node(doc, metadata, singular, item) + result.appendChild(node) + #TODO(bcwaldon): accomplish this without a type-check + elif type(data) is dict: + collections = metadata.get('dict_collections', {}) + if nodename in collections: + metadata = collections[nodename] + for k, v in data.items(): + node = doc.createElement(metadata['item_name']) + node.setAttribute(metadata['item_key'], str(k)) + text = doc.createTextNode(str(v)) + node.appendChild(text) + result.appendChild(node) + return result + attrs = metadata.get('attributes', {}).get(nodename, {}) + for k, v in data.items(): + if k in attrs: + result.setAttribute(k, str(v)) + else: + node = self._to_xml_node(doc, metadata, k, v) + result.appendChild(node) + else: + # Type is atom + node = doc.createTextNode(str(data)) + result.appendChild(node) + return result + + +class ResponseSerializer(object): + """Encode the necessary pieces into a response object""" + + def __init__(self, serializers=None): + """ + :param serializers: dictionary of content-type-specific serializers + + """ + self.serializers = { + 'application/xml': XMLDictSerializer(), + 'application/json': JSONDictSerializer(), + } + self.serializers.update(serializers or {}) + + def serialize(self, response_data, content_type, action='default'): + """Serialize a dict into a string and wrap in a wsgi.Request object. + + :param response_data: dict produced by the Controller + :param content_type: expected mimetype of serialized response body + + """ + response = webob.Response() + response.headers['Content-Type'] = content_type + + serializer = self.get_serializer(content_type) + response.body = serializer.serialize(response_data, action) + + return response + + def get_serializer(self, content_type): + try: + return self.serializers[content_type] + except (KeyError, TypeError): + raise exception.InvalidContentType(content_type=content_type) + + +class Resource(wsgi.Application): + """WSGI app that handles (de)serialization and controller dispatch. + + WSGI app that reads routing information supplied by RoutesMiddleware + and calls the requested action method upon its controller. All + controller action methods must accept a 'req' argument, which is the + incoming wsgi.Request. If the operation is a PUT or POST, the controller + method must also accept a 'body' argument (the deserialized request body). + They may raise a webob.exc exception or return a dict, which will be + serialized by requested content type. + + """ + def __init__(self, controller, serializers=None, deserializers=None): + """ + :param controller: object that implement methods created by routes lib + :param serializers: dict of content-type specific text serializers + :param deserializers: dict of content-type specific text deserializers + + """ + self.controller = controller + self.serializer = ResponseSerializer(serializers) + self.deserializer = RequestDeserializer(deserializers) + + @webob.dec.wsgify(RequestClass=Request) + def __call__(self, request): + """WSGI method that controls (de)serialization and method dispatch.""" + + LOG.debug("%(method)s %(url)s" % {"method": request.method, + "url": request.url}) + + try: + action, action_args, accept = self.deserializer.deserialize( + request) + except exception.InvalidContentType: + msg = _("Unsupported Content-Type") + return webob.exc.HTTPBadRequest(explanation=msg) + except exception.MalformedRequestBody: + msg = _("Malformed request body") + return faults.Fault(webob.exc.HTTPBadRequest(explanation=msg)) + + action_result = self.dispatch(request, action, action_args) + + #TODO(bcwaldon): find a more elegant way to pass through non-dict types + if type(action_result) is dict: + response = self.serializer.serialize(action_result, accept, action) + else: + response = action_result + + try: + msg_dict = dict(url=request.url, status=response.status_int) + msg = _("%(url)s returned with HTTP %(status)d") % msg_dict + except AttributeError, e: + msg_dict = dict(url=request.url, e=e) + msg = _("%(url)s returned a fault: %(e)s" % msg_dict) + + LOG.debug(msg) + + return response + + def dispatch(self, request, action, action_args): + """Find action-spefic method on controller and call it.""" + + controller_method = getattr(self.controller, action) + return controller_method(req=request, **action_args) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index af73d8f6d..8864f825b 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -21,14 +21,17 @@ from nova import db from nova import exception from nova import flags from nova import log as logging -from nova.api.openstack import common + +from nova.compute import api as compute from nova.scheduler import api +from nova.api.openstack import create_instance_helper as helper +from nova.api.openstack import common +from nova.api.openstack import faults +from nova.api.openstack import wsgi + FLAGS = flags.FLAGS -flags.DEFINE_string('build_plan_encryption_key', - None, - '128bit (hex) encryption key for scheduler build plans.') LOG = logging.getLogger('nova.api.openstack.zones') @@ -52,12 +55,20 @@ def _scrub_zone(zone): 'deleted', 'deleted_at', 'updated_at')) -class Controller(common.OpenstackController): +def check_encryption_key(func): + def wrapped(*args, **kwargs): + if not FLAGS.build_plan_encryption_key: + raise exception.Error(_("--build_plan_encryption_key not set")) + return func(*args, **kwargs) + return wrapped + + +class Controller(object): + """Controller for Zone resources.""" - _serialization_metadata = { - 'application/xml': { - "attributes": { - "zone": ["id", "api_url", "name", "capabilities"]}}} + def __init__(self): + self.compute_api = compute.API() + self.helper = helper.CreateInstanceHelper(self) def index(self, req): """Return all zones in brief""" @@ -92,36 +103,46 @@ class Controller(common.OpenstackController): return dict(zone=_scrub_zone(zone)) def delete(self, req, id): + """Delete a child zone entry.""" zone_id = int(id) api.zone_delete(req.environ['nova.context'], zone_id) return {} - def create(self, req): + def create(self, req, body): + """Create a child zone entry.""" context = req.environ['nova.context'] - env = self._deserialize(req.body, req.get_content_type()) - zone = api.zone_create(context, env["zone"]) + zone = api.zone_create(context, body["zone"]) return dict(zone=_scrub_zone(zone)) - def update(self, req, id): + def update(self, req, id, body): + """Update a child zone entry.""" context = req.environ['nova.context'] - env = self._deserialize(req.body, req.get_content_type()) zone_id = int(id) - zone = api.zone_update(context, zone_id, env["zone"]) + zone = api.zone_update(context, zone_id, body["zone"]) return dict(zone=_scrub_zone(zone)) - def select(self, req): + def boot(self, req, body): + """Creates a new server for a given user while being Zone aware. + + Returns a reservation ID (a UUID). + """ + result = None + try: + extra_values, result = self.helper.create_instance(req, body, + self.compute_api.create_all_at_once) + except faults.Fault, f: + return f + + reservation_id = result + return {'reservation_id': reservation_id} + + @check_encryption_key + def select(self, req, body): """Returns a weighted list of costs to create instances of desired capabilities.""" ctx = req.environ['nova.context'] - qs = req.environ['QUERY_STRING'] - param_dict = urlparse.parse_qs(qs) - param_dict.pop("fresh", None) - # parse_qs returns a dict where the values are lists, - # since query strings can have multiple values for the - # same key. We need to convert that to single values. - for key in param_dict: - param_dict[key] = param_dict[key][0] - build_plan = api.select(ctx, specs=param_dict) + specs = json.loads(body) + build_plan = api.select(ctx, specs=specs) cooked = self._scrub_build_plan(build_plan) return {"weights": cooked} @@ -129,9 +150,6 @@ class Controller(common.OpenstackController): """Remove all the confidential data and return a sanitized version of the build plan. Include an encrypted full version of the weighting entry so we can get back to it later.""" - if not FLAGS.build_plan_encryption_key: - raise exception.FlagNotSet(flag='build_plan_encryption_key') - encryptor = crypto.encryptor(FLAGS.build_plan_encryption_key) cooked = [] for entry in build_plan: @@ -140,3 +158,52 @@ class Controller(common.OpenstackController): cooked.append(dict(weight=entry['weight'], blob=cipher_text)) return cooked + + def _image_ref_from_req_data(self, data): + return data['server']['imageId'] + + def _flavor_id_from_req_data(self, data): + return data['server']['flavorId'] + + def _get_server_admin_password(self, server): + """ Determine the admin password for a server on creation """ + return self.helper._get_server_admin_password_old_style(server) + + +class ControllerV11(object): + """Controller for 1.1 Zone resources.""" + + def _get_server_admin_password(self, server): + """ Determine the admin password for a server on creation """ + return self.helper._get_server_admin_password_new_style(server) + + def _image_ref_from_req_data(self, data): + return data['server']['imageRef'] + + def _flavor_id_from_req_data(self, data): + return data['server']['flavorRef'] + + +def create_resource(version): + controller = { + '1.0': Controller, + '1.1': ControllerV11, + }[version]() + + metadata = { + "attributes": { + "zone": ["id", "api_url", "name", "capabilities"], + }, + } + + serializers = { + 'application/xml': wsgi.XMLDictSerializer(xmlns=wsgi.XMLNS_V10, + metadata=metadata), + } + + deserializers = { + 'application/xml': helper.ServerXMLDeserializer(), + } + + return wsgi.Resource(controller, serializers=serializers, + deserializers=deserializers) diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 3f8432851..e9532473d 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -24,6 +24,7 @@ other backends by creating another class that exposes the same public methods. """ +import functools import sys from nova import exception @@ -68,6 +69,12 @@ flags.DEFINE_string('ldap_developer', LOG = logging.getLogger("nova.ldapdriver") +if FLAGS.memcached_servers: + import memcache +else: + from nova import fakememcache as memcache + + # TODO(vish): make an abstract base class with the same public methods # to define a set interface for AuthDrivers. I'm delaying # creating this now because I'm expecting an auth refactor @@ -85,6 +92,7 @@ def _clean(attr): def sanitize(fn): """Decorator to sanitize all args""" + @functools.wraps(fn) def _wrapped(self, *args, **kwargs): args = [_clean(x) for x in args] kwargs = dict((k, _clean(v)) for (k, v) in kwargs) @@ -103,29 +111,56 @@ class LdapDriver(object): isadmin_attribute = 'isNovaAdmin' project_attribute = 'owner' project_objectclass = 'groupOfNames' + conn = None + mc = None def __init__(self): """Imports the LDAP module""" self.ldap = __import__('ldap') - self.conn = None if FLAGS.ldap_schema_version == 1: LdapDriver.project_pattern = '(objectclass=novaProject)' LdapDriver.isadmin_attribute = 'isAdmin' LdapDriver.project_attribute = 'projectManager' LdapDriver.project_objectclass = 'novaProject' + self.__cache = None + if LdapDriver.conn is None: + LdapDriver.conn = self.ldap.initialize(FLAGS.ldap_url) + LdapDriver.conn.simple_bind_s(FLAGS.ldap_user_dn, + FLAGS.ldap_password) + if LdapDriver.mc is None: + LdapDriver.mc = memcache.Client(FLAGS.memcached_servers, debug=0) def __enter__(self): - """Creates the connection to LDAP""" - self.conn = self.ldap.initialize(FLAGS.ldap_url) - self.conn.simple_bind_s(FLAGS.ldap_user_dn, FLAGS.ldap_password) + # TODO(yorik-sar): Should be per-request cache, not per-driver-request + self.__cache = {} return self def __exit__(self, exc_type, exc_value, traceback): - """Destroys the connection to LDAP""" - self.conn.unbind_s() + self.__cache = None return False + def __local_cache(key_fmt): # pylint: disable=E0213 + """Wrap function to cache it's result in self.__cache. + Works only with functions with one fixed argument. + """ + def do_wrap(fn): + @functools.wraps(fn) + def inner(self, arg, **kwargs): + cache_key = key_fmt % (arg,) + try: + res = self.__cache[cache_key] + LOG.debug('Local cache hit for %s by key %s' % + (fn.__name__, cache_key)) + return res + except KeyError: + res = fn(self, arg, **kwargs) + self.__cache[cache_key] = res + return res + return inner + return do_wrap + @sanitize + @__local_cache('uid_user-%s') def get_user(self, uid): """Retrieve user by id""" attr = self.__get_ldap_user(uid) @@ -134,15 +169,31 @@ class LdapDriver(object): @sanitize def get_user_from_access_key(self, access): """Retrieve user by access key""" + cache_key = 'uak_dn_%s' % (access,) + user_dn = self.mc.get(cache_key) + if user_dn: + user = self.__to_user( + self.__find_object(user_dn, scope=self.ldap.SCOPE_BASE)) + if user: + if user['access'] == access: + return user + else: + self.mc.set(cache_key, None) query = '(accessKey=%s)' % access dn = FLAGS.ldap_user_subtree - return self.__to_user(self.__find_object(dn, query)) + user_obj = self.__find_object(dn, query) + user = self.__to_user(user_obj) + if user: + self.mc.set(cache_key, user_obj['dn'][0]) + return user @sanitize + @__local_cache('pid_project-%s') def get_project(self, pid): """Retrieve project by id""" - dn = self.__project_to_dn(pid) - attr = self.__find_object(dn, LdapDriver.project_pattern) + dn = self.__project_to_dn(pid, search=False) + attr = self.__find_object(dn, LdapDriver.project_pattern, + scope=self.ldap.SCOPE_BASE) return self.__to_project(attr) @sanitize @@ -395,6 +446,7 @@ class LdapDriver(object): """Check if project exists""" return self.get_project(project_id) is not None + @__local_cache('uid_attrs-%s') def __get_ldap_user(self, uid): """Retrieve LDAP user entry by id""" dn = FLAGS.ldap_user_subtree @@ -426,12 +478,20 @@ class LdapDriver(object): if scope is None: # One of the flags is 0! scope = self.ldap.SCOPE_SUBTREE + if query is None: + query = "(objectClass=*)" try: res = self.conn.search_s(dn, scope, query) except self.ldap.NO_SUCH_OBJECT: return [] # Just return the attributes - return [attributes for dn, attributes in res] + # FIXME(yorik-sar): Whole driver should be refactored to + # prevent this hack + res1 = [] + for dn, attrs in res: + attrs['dn'] = [dn] + res1.append(attrs) + return res1 def __find_role_dns(self, tree): """Find dns of role objects in given tree""" @@ -564,6 +624,7 @@ class LdapDriver(object): 'description': attr.get('description', [None])[0], 'member_ids': [self.__dn_to_uid(x) for x in member_dns]} + @__local_cache('uid_dn-%s') def __uid_to_dn(self, uid, search=True): """Convert uid to dn""" # By default return a generated DN @@ -576,6 +637,7 @@ class LdapDriver(object): userdn = user[0] return userdn + @__local_cache('pid_dn-%s') def __project_to_dn(self, pid, search=True): """Convert pid to dn""" # By default return a generated DN @@ -603,16 +665,18 @@ class LdapDriver(object): else: return None + @__local_cache('dn_uid-%s') def __dn_to_uid(self, dn): """Convert user dn to uid""" query = '(objectclass=novaUser)' - user = self.__find_object(dn, query) + user = self.__find_object(dn, query, scope=self.ldap.SCOPE_BASE) return user[FLAGS.ldap_user_id_attribute][0] class FakeLdapDriver(LdapDriver): """Fake Ldap Auth driver""" - def __init__(self): # pylint: disable=W0231 - __import__('nova.auth.fakeldap') - self.ldap = sys.modules['nova.auth.fakeldap'] + def __init__(self): + import nova.auth.fakeldap + sys.modules['ldap'] = nova.auth.fakeldap + super(FakeLdapDriver, self).__init__() diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 07235a2a7..98c7dd263 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -73,6 +73,12 @@ flags.DEFINE_string('auth_driver', 'nova.auth.dbdriver.DbDriver', LOG = logging.getLogger('nova.auth.manager') +if FLAGS.memcached_servers: + import memcache +else: + from nova import fakememcache as memcache + + class AuthBase(object): """Base class for objects relating to auth @@ -206,6 +212,7 @@ class AuthManager(object): """ _instance = None + mc = None def __new__(cls, *args, **kwargs): """Returns the AuthManager singleton""" @@ -222,13 +229,8 @@ class AuthManager(object): self.network_manager = utils.import_object(FLAGS.network_manager) if driver or not getattr(self, 'driver', None): self.driver = utils.import_class(driver or FLAGS.auth_driver) - - if FLAGS.memcached_servers: - import memcache - else: - from nova import fakememcache as memcache - self.mc = memcache.Client(FLAGS.memcached_servers, - debug=0) + if AuthManager.mc is None: + AuthManager.mc = memcache.Client(FLAGS.memcached_servers, debug=0) def authenticate(self, access, signature, params, verb='GET', server_string='127.0.0.1:8773', path='/', diff --git a/nova/auth/novarc.template b/nova/auth/novarc.template index cda2ecc28..d05c099d7 100644 --- a/nova/auth/novarc.template +++ b/nova/auth/novarc.template @@ -1,4 +1,6 @@ -NOVA_KEY_DIR=$(pushd $(dirname $BASH_SOURCE)>/dev/null; pwd; popd>/dev/null) +NOVARC=$(readlink -f "${BASH_SOURCE:-${0}}" 2>/dev/null) || + NOVARC=$(python -c 'import os,sys; print os.path.abspath(os.path.realpath(sys.argv[1]))' "${BASH_SOURCE:-${0}}") +NOVA_KEY_DIR=${NOVARC%%/*} export EC2_ACCESS_KEY="%(access)s:%(project)s" export EC2_SECRET_KEY="%(secret)s" export EC2_URL="%(ec2)s" @@ -12,4 +14,5 @@ alias ec2-bundle-image="ec2-bundle-image --cert ${EC2_CERT} --privatekey ${EC2_P alias ec2-upload-bundle="ec2-upload-bundle -a ${EC2_ACCESS_KEY} -s ${EC2_SECRET_KEY} --url ${S3_URL} --ec2cert ${NOVA_CERT}" export NOVA_API_KEY="%(access)s" export NOVA_USERNAME="%(user)s" +export NOVA_PROJECT_ID="%(project)s" export NOVA_URL="%(os)s" diff --git a/nova/compute/api.py b/nova/compute/api.py index 7e2494781..af18741b6 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -18,7 +18,6 @@ """Handles all requests relating to instances (guest vms).""" -import datetime import eventlet import re import time @@ -26,6 +25,7 @@ import time from nova import db from nova import exception from nova import flags +import nova.image from nova import log as logging from nova import network from nova import quota @@ -34,6 +34,7 @@ from nova import utils from nova import volume from nova.compute import instance_types from nova.compute import power_state +from nova.compute.utils import terminate_volumes from nova.scheduler import api as scheduler_api from nova.db import base @@ -52,15 +53,27 @@ def generate_default_hostname(instance_id): return str(instance_id) +def _is_able_to_shutdown(instance, instance_id): + states = {'terminating': "Instance %s is already being terminated", + 'migrating': "Instance %s is being migrated", + 'stopping': "Instance %s is being stopped"} + msg = states.get(instance['state_description']) + if msg: + LOG.warning(_(msg), instance_id) + return False + + return True + + class API(base.Base): """API for interacting with the compute manager.""" def __init__(self, image_service=None, network_api=None, volume_api=None, hostname_factory=generate_default_hostname, **kwargs): - if not image_service: - image_service = utils.import_object(FLAGS.image_service) - self.image_service = image_service + self.image_service = image_service or \ + nova.image.get_default_image_service() + if not network_api: network_api = network.API() self.network_api = network_api @@ -91,18 +104,18 @@ class API(base.Base): """Enforce quota limits on injected files. Raises a QuotaError if any limit is exceeded. - """ if injected_files is None: return - limit = quota.allowed_injected_files(context) + limit = quota.allowed_injected_files(context, len(injected_files)) if len(injected_files) > limit: raise quota.QuotaError(code="OnsetFileLimitExceeded") path_limit = quota.allowed_injected_file_path_bytes(context) - content_limit = quota.allowed_injected_file_content_bytes(context) for path, content in injected_files: if len(path) > path_limit: raise quota.QuotaError(code="OnsetFilePathLimitExceeded") + content_limit = quota.allowed_injected_file_content_bytes( + context, len(content)) if len(content) > content_limit: raise quota.QuotaError(code="OnsetFileContentLimitExceeded") @@ -128,18 +141,17 @@ class API(base.Base): LOG.warn(msg) raise quota.QuotaError(msg, "MetadataLimitExceeded") - def create(self, context, instance_type, - image_id, kernel_id=None, ramdisk_id=None, + def _check_create_parameters(self, context, instance_type, + image_href, kernel_id=None, ramdisk_id=None, min_count=1, max_count=1, display_name='', display_description='', key_name=None, key_data=None, security_group='default', availability_zone=None, user_data=None, metadata={}, - injected_files=None): - """Create the number and type of instances requested. - - Verifies that quota and other arguments are valid. + injected_files=None, admin_password=None, zone_blob=None, + reservation_id=None): + """Verify all the input parameters regardless of the provisioning + strategy being performed.""" - """ if not instance_type: instance_type = instance_types.get_default_instance_type() @@ -149,18 +161,29 @@ class API(base.Base): pid = context.project_id LOG.warn(_("Quota exceeeded for %(pid)s," " tried to run %(min_count)s instances") % locals()) - raise quota.QuotaError(_("Instance quota exceeded. You can only " - "run %s more instances of this type.") % - num_instances, "InstanceLimitExceeded") + if num_instances <= 0: + message = _("Instance quota exceeded. You cannot run any " + "more instances of this type.") + else: + message = _("Instance quota exceeded. You can only run %s " + "more instances of this type.") % num_instances + raise quota.QuotaError(message, "InstanceLimitExceeded") self._check_metadata_properties_quota(context, metadata) self._check_injected_file_quota(context, injected_files) - image = self.image_service.show(context, image_id) + (image_service, image_id) = nova.image.get_image_service(image_href) + image = image_service.show(context, image_id) os_type = None if 'properties' in image and 'os_type' in image['properties']: os_type = image['properties']['os_type'] + architecture = None + if 'properties' in image and 'arch' in image['properties']: + architecture = image['properties']['arch'] + vm_mode = None + if 'properties' in image and 'vm_mode' in image['properties']: + vm_mode = image['properties']['vm_mode'] if kernel_id is None: kernel_id = image['properties'].get('kernel_id', None) @@ -176,9 +199,9 @@ class API(base.Base): logging.debug("Using Kernel=%s, Ramdisk=%s" % (kernel_id, ramdisk_id)) if kernel_id: - self.image_service.show(context, kernel_id) + image_service.show(context, kernel_id) if ramdisk_id: - self.image_service.show(context, ramdisk_id) + image_service.show(context, ramdisk_id) if security_group is None: security_group = ['default'] @@ -197,9 +220,12 @@ class API(base.Base): key_pair = db.key_pair_get(context, context.user_id, key_name) key_data = key_pair['public_key'] + if reservation_id is None: + reservation_id = utils.generate_uid('r') + base_options = { - 'reservation_id': utils.generate_uid('r'), - 'image_id': image_id, + 'reservation_id': reservation_id, + 'image_ref': image_href, 'kernel_id': kernel_id or '', 'ramdisk_id': ramdisk_id or '', 'state': 0, @@ -219,55 +245,170 @@ class API(base.Base): 'locked': False, 'metadata': metadata, 'availability_zone': availability_zone, - 'os_type': os_type} + 'os_type': os_type, + 'architecture': architecture, + 'vm_mode': vm_mode} + + return (num_instances, base_options, security_groups) + + def create_db_entry_for_new_instance(self, context, base_options, + security_groups, block_device_mapping, num=1): + """Create an entry in the DB for this new instance, + including any related table updates (such as security + groups, MAC address, etc). This will called by create() + in the majority of situations, but all-at-once style + Schedulers may initiate the call.""" + instance = dict(mac_address=utils.generate_mac(), + launch_index=num, + **base_options) + instance = self.db.instance_create(context, instance) + instance_id = instance['id'] + elevated = context.elevated() - instances = [] - LOG.debug(_("Going to run %s instances..."), num_instances) - for num in range(num_instances): - instance = dict(mac_address=utils.generate_mac(), - launch_index=num, - **base_options) - instance = self.db.instance_create(context, instance) - instance_id = instance['id'] + if not security_groups: + security_groups = [] + for security_group_id in security_groups: + self.db.instance_add_security_group(elevated, + instance_id, + security_group_id) + + # NOTE(yamahata) + # tell vm driver to attach volume at boot time by updating + # BlockDeviceMapping + for bdm in block_device_mapping: + LOG.debug(_('bdm %s'), bdm) + assert 'device_name' in bdm + values = { + 'instance_id': instance_id, + 'device_name': bdm['device_name'], + 'delete_on_termination': bdm.get('delete_on_termination'), + 'virtual_name': bdm.get('virtual_name'), + 'snapshot_id': bdm.get('snapshot_id'), + 'volume_id': bdm.get('volume_id'), + 'volume_size': bdm.get('volume_size'), + 'no_device': bdm.get('no_device')} + self.db.block_device_mapping_create(elevated, values) + + # Set sane defaults if not specified + updates = dict(hostname=self.hostname_factory(instance_id)) + if (not hasattr(instance, 'display_name') or + instance.display_name is None): + updates['display_name'] = "Server %s" % instance_id + + instance = self.update(context, instance_id, **updates) - elevated = context.elevated() - if not security_groups: - security_groups = [] - for security_group_id in security_groups: - self.db.instance_add_security_group(elevated, - instance_id, - security_group_id) - - # Set sane defaults if not specified - updates = dict(hostname=self.hostname_factory(instance_id)) - if (not hasattr(instance, 'display_name') or - instance.display_name is None): - updates['display_name'] = "Server %s" % instance_id - - instance = self.update(context, instance_id, **updates) - instances.append(instance) + for group_id in security_groups: + self.trigger_security_group_members_refresh(elevated, group_id) - pid = context.project_id - uid = context.user_id + return instance + + def _ask_scheduler_to_create_instance(self, context, base_options, + instance_type, zone_blob, + availability_zone, injected_files, + admin_password, + instance_id=None, num_instances=1): + """Send the run_instance request to the schedulers for processing.""" + pid = context.project_id + uid = context.user_id + if instance_id: + LOG.debug(_("Casting to scheduler for %(pid)s/%(uid)s's" + " instance %(instance_id)s (single-shot)") % locals()) + else: LOG.debug(_("Casting to scheduler for %(pid)s/%(uid)s's" - " instance %(instance_id)s") % locals()) + " (all-at-once)") % locals()) + + filter_class = 'nova.scheduler.host_filter.InstanceTypeFilter' + request_spec = { + 'instance_properties': base_options, + 'instance_type': instance_type, + 'filter': filter_class, + 'blob': zone_blob, + 'num_instances': num_instances, + } - # NOTE(sandy): For now we're just going to pass in the - # instance_type record to the scheduler. In a later phase - # we'll be ripping this whole for-loop out and deferring the - # creation of the Instance record. At that point all this will - # change. - rpc.cast(context, - FLAGS.scheduler_topic, - {"method": "run_instance", - "args": {"topic": FLAGS.compute_topic, - "instance_id": instance_id, - "instance_type": instance_type, - "availability_zone": availability_zone, - "injected_files": injected_files}}) + rpc.cast(context, + FLAGS.scheduler_topic, + {"method": "run_instance", + "args": {"topic": FLAGS.compute_topic, + "instance_id": instance_id, + "request_spec": request_spec, + "availability_zone": availability_zone, + "admin_password": admin_password, + "injected_files": injected_files}}) - for group_id in security_groups: - self.trigger_security_group_members_refresh(elevated, group_id) + def create_all_at_once(self, context, instance_type, + image_href, kernel_id=None, ramdisk_id=None, + min_count=1, max_count=1, + display_name='', display_description='', + key_name=None, key_data=None, security_group='default', + availability_zone=None, user_data=None, metadata={}, + injected_files=None, admin_password=None, zone_blob=None, + reservation_id=None): + """Provision the instances by passing the whole request to + the Scheduler for execution. Returns a Reservation ID + related to the creation of all of these instances.""" + num_instances, base_options, security_groups = \ + self._check_create_parameters( + context, instance_type, + image_href, kernel_id, ramdisk_id, + min_count, max_count, + display_name, display_description, + key_name, key_data, security_group, + availability_zone, user_data, metadata, + injected_files, admin_password, zone_blob, + reservation_id) + + self._ask_scheduler_to_create_instance(context, base_options, + instance_type, zone_blob, + availability_zone, injected_files, + admin_password, + num_instances=num_instances) + + return base_options['reservation_id'] + + def create(self, context, instance_type, + image_href, kernel_id=None, ramdisk_id=None, + min_count=1, max_count=1, + display_name='', display_description='', + key_name=None, key_data=None, security_group='default', + availability_zone=None, user_data=None, metadata={}, + injected_files=None, admin_password=None, zone_blob=None, + reservation_id=None, block_device_mapping=None): + """ + Provision the instances by sending off a series of single + instance requests to the Schedulers. This is fine for trival + Scheduler drivers, but may remove the effectiveness of the + more complicated drivers. + + Returns a list of instance dicts. + """ + + num_instances, base_options, security_groups = \ + self._check_create_parameters( + context, instance_type, + image_href, kernel_id, ramdisk_id, + min_count, max_count, + display_name, display_description, + key_name, key_data, security_group, + availability_zone, user_data, metadata, + injected_files, admin_password, zone_blob, + reservation_id) + + block_device_mapping = block_device_mapping or [] + instances = [] + LOG.debug(_("Going to run %s instances..."), num_instances) + for num in range(num_instances): + instance = self.create_db_entry_for_new_instance(context, + base_options, security_groups, + block_device_mapping, num=num) + instances.append(instance) + instance_id = instance['id'] + + self._ask_scheduler_to_create_instance(context, base_options, + instance_type, zone_blob, + availability_zone, injected_files, + admin_password, + instance_id=instance_id) return [dict(x.iteritems()) for x in instances] @@ -287,7 +428,6 @@ class API(base.Base): already exist. :param context: the security context - """ try: db.security_group_get_by_name(context, context.project_id, @@ -320,7 +460,6 @@ class API(base.Base): Sends an update request to each compute node for whom this is relevant. - """ # First, we get the security group rules that reference this group as # the grantee.. @@ -357,6 +496,16 @@ class API(base.Base): {"method": "refresh_security_group_members", "args": {"security_group_id": group_id}}) + def trigger_provider_fw_rules_refresh(self, context): + """Called when a rule is added to or removed from a security_group""" + + hosts = [x['host'] for (x, idx) + in db.service_get_all_compute_sorted(context)] + for host in hosts: + rpc.cast(context, + self.db.queue_get_for(context, FLAGS.compute_topic, host), + {'method': 'refresh_provider_fw_rules', 'args': {}}) + def update(self, context, instance_id, **kwargs): """Updates the instance in the datastore. @@ -367,48 +516,92 @@ class API(base.Base): updated :returns: None - """ rv = self.db.instance_update(context, instance_id, kwargs) return dict(rv.iteritems()) - @scheduler_api.reroute_compute("delete") - def delete(self, context, instance_id): - """Terminate an instance.""" - LOG.debug(_("Going to try to terminate %s"), instance_id) + def _get_instance(self, context, instance_id, action_str): try: - instance = self.get(context, instance_id) + return self.get(context, instance_id) except exception.NotFound: - LOG.warning(_("Instance %s was not found during terminate"), - instance_id) + LOG.warning(_("Instance %(instance_id)s was not found during " + "%(action_str)s") % + {'instance_id': instance_id, 'action_str': action_str}) raise - if instance['state_description'] == 'terminating': - LOG.warning(_("Instance %s is already being terminated"), - instance_id) - return + @scheduler_api.reroute_compute("delete") + def delete(self, context, instance_id): + """Terminate an instance.""" + LOG.debug(_("Going to try to terminate %s"), instance_id) + instance = self._get_instance(context, instance_id, 'terminating') - if instance['state_description'] == 'migrating': - LOG.warning(_("Instance %s is being migrated"), instance_id) + if not _is_able_to_shutdown(instance, instance_id): return self.update(context, instance['id'], state_description='terminating', state=0, - terminated_at=datetime.datetime.utcnow()) + terminated_at=utils.utcnow()) host = instance['host'] if host: self._cast_compute_message('terminate_instance', context, instance_id, host) else: + terminate_volumes(self.db, context, instance_id) self.db.instance_destroy(context, instance_id) + @scheduler_api.reroute_compute("stop") + def stop(self, context, instance_id): + """Stop an instance.""" + LOG.debug(_("Going to try to stop %s"), instance_id) + + instance = self._get_instance(context, instance_id, 'stopping') + if not _is_able_to_shutdown(instance, instance_id): + return + + self.update(context, + instance['id'], + state_description='stopping', + state=power_state.NOSTATE, + terminated_at=utils.utcnow()) + + host = instance['host'] + if host: + self._cast_compute_message('stop_instance', context, + instance_id, host) + + def start(self, context, instance_id): + """Start an instance.""" + LOG.debug(_("Going to try to start %s"), instance_id) + instance = self._get_instance(context, instance_id, 'starting') + if instance['state_description'] != 'stopped': + _state_description = instance['state_description'] + LOG.warning(_("Instance %(instance_id)s is not " + "stopped(%(_state_description)s)") % locals()) + return + + # TODO(yamahata): injected_files isn't supported right now. + # It is used only for osapi. not for ec2 api. + # availability_zone isn't used by run_instance. + rpc.cast(context, + FLAGS.scheduler_topic, + {"method": "start_instance", + "args": {"topic": FLAGS.compute_topic, + "instance_id": instance_id}}) + def get(self, context, instance_id): """Get a single instance with the given instance_id.""" - rv = self.db.instance_get(context, instance_id) - return dict(rv.iteritems()) + # NOTE(sirp): id used to be exclusively integer IDs; now we're + # accepting both UUIDs and integer IDs. The handling of this + # is done in db/sqlalchemy/api/instance_get + if utils.is_uuid_like(instance_id): + uuid = instance_id + instance = self.db.instance_get_by_uuid(context, uuid) + else: + instance = self.db.instance_get(context, instance_id) + return dict(instance.iteritems()) @scheduler_api.reroute_compute("get") def routing_get(self, context, instance_id): @@ -417,21 +610,37 @@ class API(base.Base): Use this method instead of get() if this is the only operation you intend to to. It will route to novaclient.get if the instance is not found. - """ return self.get(context, instance_id) + def get_all_across_zones(self, context, reservation_id): + """Get all instances with this reservation_id, across + all available Zones (if any). + """ + context = context.elevated() + instances = self.db.instance_get_all_by_reservation( + context, reservation_id) + + children = scheduler_api.call_zone_method(context, "list", + novaclient_collection_name="servers", + reservation_id=reservation_id) + + for zone, servers in children: + for server in servers: + # Results are ready to send to user. No need to scrub. + server._info['_is_precooked'] = True + instances.append(server._info) + return instances + def get_all(self, context, project_id=None, reservation_id=None, fixed_ip=None): """Get all instances filtered by one of the given parameters. If there is no filter and the context is an admin, it will retreive all instances in the system. - """ if reservation_id is not None: - return self.db.instance_get_all_by_reservation( - context, reservation_id) + return self.get_all_across_zones(context, reservation_id) if fixed_ip is not None: return self.db.fixed_ip_get_instance(context, fixed_ip) @@ -457,7 +666,6 @@ class API(base.Base): compute worker :returns: None - """ if not params: params = {} @@ -503,25 +711,17 @@ class API(base.Base): raise exception.Error(_("Unable to find host for Instance %s") % instance_id) - def _set_admin_password(self, context, instance_id, password): - """Set the root/admin password for the given instance.""" - host = self._find_host(context, instance_id) - - rpc.cast(context, - self.db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "set_admin_password", - "args": {"instance_id": instance_id, "new_pass": password}}) - - def snapshot(self, context, instance_id, name): + def snapshot(self, context, instance_id, name, extra_properties=None): """Snapshot the given instance. :returns: A dict containing image metadata - """ properties = {'instance_id': str(instance_id), - 'user_id': str(context.user_id)} + 'user_id': str(context.user_id), + 'image_state': 'creating'} + properties.update(extra_properties or {}) sent_meta = {'name': name, 'is_public': False, - 'properties': properties} + 'status': 'creating', 'properties': properties} recv_meta = self.image_service.create(context, sent_meta) params = {'image_id': recv_meta['id']} self._cast_compute_message('snapshot_instance', context, instance_id, @@ -532,8 +732,8 @@ class API(base.Base): """Reboot the given instance.""" self._cast_compute_message('reboot_instance', context, instance_id) - def rebuild(self, context, instance_id, image_id, metadata=None, - files_to_inject=None): + def rebuild(self, context, instance_id, image_href, name=None, + metadata=None, files_to_inject=None): """Rebuild the given instance with the provided metadata.""" instance = db.api.instance_get(context, instance_id) @@ -541,16 +741,19 @@ class API(base.Base): msg = _("Instance already building") raise exception.BuildInProgress(msg) - metadata = metadata or {} - self._check_metadata_properties_quota(context, metadata) - files_to_inject = files_to_inject or [] self._check_injected_file_quota(context, files_to_inject) - self.db.instance_update(context, instance_id, {"metadata": metadata}) + values = {} + if metadata is not None: + self._check_metadata_properties_quota(context, metadata) + values['metadata'] = metadata + if name is not None: + values['display_name'] = name + self.db.instance_update(context, instance_id, values) rebuild_params = { - "image_id": image_id, + "image_ref": image_href, "injected_files": files_to_inject, } @@ -665,8 +868,12 @@ class API(base.Base): def set_admin_password(self, context, instance_id, password=None): """Set the root/admin password for the given instance.""" - eventlet.spawn_n(self._set_admin_password(context, instance_id, - password)) + host = self._find_host(context, instance_id) + + rpc.cast(context, + self.db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "set_admin_password", + "args": {"instance_id": instance_id, "new_pass": password}}) def inject_file(self, context, instance_id): """Write a file to the given instance.""" diff --git a/nova/compute/instance_types.py b/nova/compute/instance_types.py index 1275a6fdd..1d246e445 100644 --- a/nova/compute/instance_types.py +++ b/nova/compute/instance_types.py @@ -114,7 +114,7 @@ def get_instance_type(id): ctxt = context.get_admin_context() return db.instance_type_get_by_id(ctxt, id) except exception.DBError: - raise exception.ApiError(_("Unknown instance type: %s") % name) + raise exception.ApiError(_("Unknown instance type: %s") % id) def get_instance_type_by_name(name): diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 11565c25e..5aed2c677 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -35,7 +35,6 @@ terminating it. """ -import datetime import os import socket import sys @@ -54,6 +53,7 @@ from nova import rpc from nova import utils from nova import volume from nova.compute import power_state +from nova.compute.utils import terminate_volumes from nova.virt import driver @@ -159,12 +159,12 @@ class ComputeManager(manager.SchedulerDependentManager): def _update_launched_at(self, context, instance_id, launched_at=None): """Update the launched_at parameter of the given instance.""" - data = {'launched_at': launched_at or datetime.datetime.utcnow()} + data = {'launched_at': launched_at or utils.utcnow()} self.db.instance_update(context, instance_id, data) - def _update_image_id(self, context, instance_id, image_id): + def _update_image_ref(self, context, instance_id, image_ref): """Update the image_id for the given instance.""" - data = {'image_id': image_id} + data = {'image_ref': image_ref} self.db.instance_update(context, instance_id, data) def get_console_topic(self, context, **kwargs): @@ -216,11 +216,72 @@ class ComputeManager(manager.SchedulerDependentManager): return self.driver.refresh_security_group_members(security_group_id) @exception.wrap_exception - def run_instance(self, context, instance_id, **kwargs): + def refresh_provider_fw_rules(self, context, **_kwargs): + """This call passes straight through to the virtualization driver.""" + return self.driver.refresh_provider_fw_rules() + + def _setup_block_device_mapping(self, context, instance_id): + """setup volumes for block device mapping""" + self.db.instance_set_state(context, + instance_id, + power_state.NOSTATE, + 'block_device_mapping') + + volume_api = volume.API() + block_device_mapping = [] + for bdm in self.db.block_device_mapping_get_all_by_instance( + context, instance_id): + LOG.debug(_("setting up bdm %s"), bdm) + if ((bdm['snapshot_id'] is not None) and + (bdm['volume_id'] is None)): + # TODO(yamahata): default name and description + vol = volume_api.create(context, bdm['volume_size'], + bdm['snapshot_id'], '', '') + # TODO(yamahata): creating volume simultaneously + # reduces creation time? + volume_api.wait_creation(context, vol['id']) + self.db.block_device_mapping_update( + context, bdm['id'], {'volume_id': vol['id']}) + bdm['volume_id'] = vol['id'] + + if not ((bdm['snapshot_id'] is None) or + (bdm['volume_id'] is not None)): + LOG.error(_('corrupted state of block device mapping ' + 'id: %(id)s ' + 'snapshot: %(snapshot_id) volume: %(vollume_id)') % + {'id': bdm['id'], + 'snapshot_id': bdm['snapshot'], + 'volume_id': bdm['volume_id']}) + raise exception.ApiError(_('broken block device mapping %d') % + bdm['id']) + + if bdm['volume_id'] is not None: + volume_api.check_attach(context, + volume_id=bdm['volume_id']) + dev_path = self._attach_volume_boot(context, instance_id, + bdm['volume_id'], + bdm['device_name']) + block_device_mapping.append({'device_path': dev_path, + 'mount_device': + bdm['device_name']}) + elif bdm['virtual_name'] is not None: + # TODO(yamahata): ephemeral/swap device support + LOG.debug(_('block_device_mapping: ' + 'ephemeral device is not supported yet')) + else: + # TODO(yamahata): NoDevice support + assert bdm['no_device'] + LOG.debug(_('block_device_mapping: ' + 'no device is not supported yet')) + + return block_device_mapping + + def _run_instance(self, context, instance_id, **kwargs): """Launch a new instance with specified options.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) instance_ref.injected_files = kwargs.get('injected_files', []) + instance_ref.admin_pass = kwargs.get('admin_password', None) if instance_ref['name'] in self.driver.list_instances(): raise exception.Error(_("Instance has already been created")) LOG.audit(_("instance %s: starting..."), instance_id, @@ -234,55 +295,80 @@ class ComputeManager(manager.SchedulerDependentManager): power_state.NOSTATE, 'networking') - is_vpn = instance_ref['image_id'] == str(FLAGS.vpn_image_id) - # NOTE(vish): This could be a cast because we don't do anything - # with the address currently, but I'm leaving it as - # a call to ensure that network setup completes. We - # will eventually also need to save the address here. - if not FLAGS.stub_network: - address = rpc.call(context, - self.get_network_topic(context), - {"method": "allocate_fixed_ip", - "args": {"instance_id": instance_id, - "vpn": is_vpn}}) - - self.network_manager.setup_compute_network(context, - instance_id) - - # TODO(vish) check to make sure the availability zone matches - self._update_state(context, instance_id, power_state.BUILDING) - + is_vpn = instance_ref['image_ref'] == str(FLAGS.vpn_image_id) try: - self.driver.spawn(instance_ref) - except Exception as ex: # pylint: disable=W0702 - msg = _("Instance '%(instance_id)s' failed to spawn. Is " - "virtualization enabled in the BIOS? Details: " - "%(ex)s") % locals() - LOG.exception(msg) - - if not FLAGS.stub_network and FLAGS.auto_assign_floating_ip: - public_ip = self.network_api.allocate_floating_ip(context) - - self.db.floating_ip_set_auto_assigned(context, public_ip) - fixed_ip = self.db.fixed_ip_get_by_address(context, address) - floating_ip = self.db.floating_ip_get_by_address(context, - public_ip) - - self.network_api.associate_floating_ip(context, - floating_ip, - fixed_ip, - affect_auto_assigned=True) + # NOTE(vish): This could be a cast because we don't do anything + # with the address currently, but I'm leaving it as + # a call to ensure that network setup completes. We + # will eventually also need to save the address here. + if not FLAGS.stub_network: + address = rpc.call(context, + self.get_network_topic(context), + {"method": "allocate_fixed_ip", + "args": {"instance_id": instance_id, + "vpn": is_vpn}}) - self._update_launched_at(context, instance_id) - self._update_state(context, instance_id) + self.network_manager.setup_compute_network(context, + instance_id) + + block_device_mapping = self._setup_block_device_mapping( + context, + instance_id) + + # TODO(vish) check to make sure the availability zone matches + self._update_state(context, instance_id, power_state.BUILDING) + + try: + self.driver.spawn(instance_ref, + block_device_mapping=block_device_mapping) + except Exception as ex: # pylint: disable=W0702 + msg = _("Instance '%(instance_id)s' failed to spawn. Is " + "virtualization enabled in the BIOS? Details: " + "%(ex)s") % locals() + LOG.exception(msg) + + if not FLAGS.stub_network and FLAGS.auto_assign_floating_ip: + public_ip = self.network_api.allocate_floating_ip(context) + + self.db.floating_ip_set_auto_assigned(context, public_ip) + fixed_ip = self.db.fixed_ip_get_by_address(context, address) + floating_ip = self.db.floating_ip_get_by_address(context, + public_ip) + + self.network_api.associate_floating_ip( + context, + floating_ip, + fixed_ip, + affect_auto_assigned=True) + + self._update_launched_at(context, instance_id) + self._update_state(context, instance_id) + except exception.InstanceNotFound: + # FIXME(wwolf): We are just ignoring InstanceNotFound + # exceptions here in case the instance was immediately + # deleted before it actually got created. This should + # be fixed once we have no-db-messaging + pass + + @exception.wrap_exception + def run_instance(self, context, instance_id, **kwargs): + self._run_instance(context, instance_id, **kwargs) @exception.wrap_exception @checks_instance_lock - def terminate_instance(self, context, instance_id): - """Terminate an instance on this host.""" + def start_instance(self, context, instance_id): + """Starting an instance on this host.""" + # TODO(yamahata): injected_files isn't supported. + # Anyway OSAPI doesn't support stop/start yet + self._run_instance(context, instance_id) + + def _shutdown_instance(self, context, instance_id, action_str): + """Shutdown an instance on this host.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) - LOG.audit(_("Terminating instance %s"), instance_id, context=context) + LOG.audit(_("%(action_str)s instance %(instance_id)s") % + {'action_str': action_str, 'instance_id': instance_id}, + context=context) fixed_ip = instance_ref.get('fixed_ip') if not FLAGS.stub_network and fixed_ip: @@ -318,19 +404,37 @@ class ComputeManager(manager.SchedulerDependentManager): volumes = instance_ref.get('volumes') or [] for volume in volumes: - self.detach_volume(context, instance_id, volume['id']) - if instance_ref['state'] == power_state.SHUTOFF: + self._detach_volume(context, instance_id, volume['id'], False) + + if (instance_ref['state'] == power_state.SHUTOFF and + instance_ref['state_description'] != 'stopped'): self.db.instance_destroy(context, instance_id) raise exception.Error(_('trying to destroy already destroyed' ' instance: %s') % instance_id) self.driver.destroy(instance_ref) + if action_str == 'Terminating': + terminate_volumes(self.db, context, instance_id) + + @exception.wrap_exception + @checks_instance_lock + def terminate_instance(self, context, instance_id): + """Terminate an instance on this host.""" + self._shutdown_instance(context, instance_id, 'Terminating') + # TODO(ja): should we keep it in a terminated state for a bit? self.db.instance_destroy(context, instance_id) @exception.wrap_exception @checks_instance_lock - def rebuild_instance(self, context, instance_id, image_id): + def stop_instance(self, context, instance_id): + """Stopping an instance on this host.""" + self._shutdown_instance(context, instance_id, 'Stopping') + # instance state will be updated to stopped by _poll_instance_states() + + @exception.wrap_exception + @checks_instance_lock + def rebuild_instance(self, context, instance_id, **kwargs): """Destroy and re-make this instance. A 'rebuild' effectively purges all existing data from the system and @@ -338,7 +442,7 @@ class ComputeManager(manager.SchedulerDependentManager): :param context: `nova.RequestContext` object :param instance_id: Instance identifier (integer) - :param image_id: Image identifier (integer) + :param image_ref: Image identifier (href or integer) """ context = context.elevated() @@ -348,10 +452,12 @@ class ComputeManager(manager.SchedulerDependentManager): self._update_state(context, instance_id, power_state.BUILDING) self.driver.destroy(instance_ref) - instance_ref.image_id = image_id + image_ref = kwargs.get('image_ref') + instance_ref.image_ref = image_ref + instance_ref.injected_files = kwargs.get('injected_files', []) self.driver.spawn(instance_ref) - self._update_image_id(context, instance_id, image_id) + self._update_image_ref(context, instance_id, image_ref) self._update_launched_at(context, instance_id) self._update_state(context, instance_id) @@ -405,22 +511,28 @@ class ComputeManager(manager.SchedulerDependentManager): @exception.wrap_exception @checks_instance_lock def set_admin_password(self, context, instance_id, new_pass=None): - """Set the root/admin password for an instance on this host.""" + """Set the root/admin password for an instance on this host. + + This is generally only called by API password resets after an + image has been built. + """ + context = context.elevated() if new_pass is None: # Generate a random password new_pass = utils.generate_password(FLAGS.password_length) - while True: + max_tries = 10 + + for i in xrange(max_tries): instance_ref = self.db.instance_get(context, instance_id) instance_id = instance_ref["id"] instance_state = instance_ref["state"] expected_state = power_state.RUNNING if instance_state != expected_state: - time.sleep(5) - continue + raise exception.Error(_('Instance is not running')) else: try: self.driver.set_admin_password(instance_ref, new_pass) @@ -436,6 +548,12 @@ class ComputeManager(manager.SchedulerDependentManager): except Exception, e: # Catch all here because this could be anything. LOG.exception(e) + if i == max_tries - 1: + # At some point this exception may make it back + # to the API caller, and we don't want to reveal + # too much. The real exception is logged above + raise exception.Error(_('Internal error')) + time.sleep(1) continue @exception.wrap_exception @@ -458,6 +576,24 @@ class ComputeManager(manager.SchedulerDependentManager): @exception.wrap_exception @checks_instance_lock + def agent_update(self, context, instance_id, url, md5hash): + """Update agent running on an instance on this host.""" + context = context.elevated() + instance_ref = self.db.instance_get(context, instance_id) + instance_id = instance_ref['id'] + instance_state = instance_ref['state'] + expected_state = power_state.RUNNING + if instance_state != expected_state: + LOG.warn(_('trying to update agent on a non-running ' + 'instance: %(instance_id)s (state: %(instance_state)s ' + 'expected: %(expected_state)s)') % locals()) + nm = instance_ref['name'] + msg = _('instance %(nm)s: updating agent to %(url)s') % locals() + LOG.audit(msg) + self.driver.agent_update(instance_ref, url, md5hash) + + @exception.wrap_exception + @checks_instance_lock def rescue_instance(self, context, instance_id): """Rescue an instance on this host.""" context = context.elevated() @@ -679,7 +815,6 @@ class ComputeManager(manager.SchedulerDependentManager): def get_diagnostics(self, context, instance_id): """Retrieve diagnostics for an instance on this host.""" instance_ref = self.db.instance_get(context, instance_id) - if instance_ref["state"] == power_state.RUNNING: LOG.audit(_("instance %s: retrieving diagnostics"), instance_id, context=context) @@ -786,6 +921,22 @@ class ComputeManager(manager.SchedulerDependentManager): instance_ref = self.db.instance_get(context, instance_id) return self.driver.get_vnc_console(instance_ref) + def _attach_volume_boot(self, context, instance_id, volume_id, mountpoint): + """Attach a volume to an instance at boot time. So actual attach + is done by instance creation""" + + # TODO(yamahata): + # should move check_attach to volume manager? + volume.API().check_attach(context, volume_id) + + context = context.elevated() + LOG.audit(_("instance %(instance_id)s: booting with " + "volume %(volume_id)s at %(mountpoint)s") % + locals(), context=context) + dev_path = self.volume_manager.setup_compute_volume(context, volume_id) + self.db.volume_attached(context, volume_id, instance_id, mountpoint) + return dev_path + @checks_instance_lock def attach_volume(self, context, instance_id, volume_id, mountpoint): """Attach a volume to an instance.""" @@ -803,6 +954,16 @@ class ComputeManager(manager.SchedulerDependentManager): volume_id, instance_id, mountpoint) + values = { + 'instance_id': instance_id, + 'device_name': mountpoint, + 'delete_on_termination': False, + 'virtual_name': None, + 'snapshot_id': None, + 'volume_id': volume_id, + 'volume_size': None, + 'no_device': None} + self.db.block_device_mapping_create(context, values) except Exception as exc: # pylint: disable=W0702 # NOTE(vish): The inline callback eats the exception info so we # log the traceback here and reraise the same @@ -817,7 +978,7 @@ class ComputeManager(manager.SchedulerDependentManager): @exception.wrap_exception @checks_instance_lock - def detach_volume(self, context, instance_id, volume_id): + def _detach_volume(self, context, instance_id, volume_id, destroy_bdm): """Detach a volume from an instance.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) @@ -833,8 +994,15 @@ class ComputeManager(manager.SchedulerDependentManager): volume_ref['mountpoint']) self.volume_manager.remove_compute_volume(context, volume_id) self.db.volume_detached(context, volume_id) + if destroy_bdm: + self.db.block_device_mapping_destroy_by_instance_and_volume( + context, instance_id, volume_id) return True + def detach_volume(self, context, instance_id, volume_id): + """Detach a volume from an instance.""" + return self._detach_volume(context, instance_id, volume_id, True) + def remove_volume(self, context, volume_id): """Remove volume on compute host. @@ -1160,11 +1328,14 @@ class ComputeManager(manager.SchedulerDependentManager): "State=%(db_state)s, so setting state to " "shutoff.") % locals()) vm_state = power_state.SHUTOFF + if db_instance['state_description'] == 'stopping': + self.db.instance_stop(context, db_instance['id']) + continue else: vm_state = vm_instance.state vms_not_found_in_db.remove(name) - if db_instance['state_description'] == 'migrating': + if (db_instance['state_description'] in ['migrating', 'stopping']): # A situation which db record exists, but no instance" # sometimes occurs while live-migration at src compute, # this case should be ignored. diff --git a/nova/compute/monitor.py b/nova/compute/monitor.py index 3bb54a382..9d8e2a25d 100644 --- a/nova/compute/monitor.py +++ b/nova/compute/monitor.py @@ -36,6 +36,7 @@ from twisted.application import service from nova import flags from nova import log as logging +from nova import utils from nova.virt import connection as virt_connection @@ -86,7 +87,7 @@ RRD_VALUES = { ]} -utcnow = datetime.datetime.utcnow +utcnow = utils.utcnow LOG = logging.getLogger('nova.compute.monitor') diff --git a/nova/compute/utils.py b/nova/compute/utils.py new file mode 100644 index 000000000..c8cb9bab8 --- /dev/null +++ b/nova/compute/utils.py @@ -0,0 +1,29 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 VA Linux Systems Japan K.K +# Copyright (c) 2011 Isaku Yamahata +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import volume + + +def terminate_volumes(db, context, instance_id): + """delete volumes of delete_on_termination=True in block device mapping""" + volume_api = volume.API() + for bdm in db.block_device_mapping_get_all_by_instance(context, + instance_id): + #LOG.debug(_("terminating bdm %s") % bdm) + if bdm['volume_id'] and bdm['delete_on_termination']: + volume_api.delete(context, bdm['volume_id']) + db.block_device_mapping_destroy(context, bdm['id']) diff --git a/nova/console/vmrc.py b/nova/console/vmrc.py index cc8b0cdf5..fa805e019 100644 --- a/nova/console/vmrc.py +++ b/nova/console/vmrc.py @@ -119,7 +119,7 @@ class VMRCSessionConsole(VMRCConsole): """ vms = vim_session._call_method(vim_util, 'get_objects', 'VirtualMachine', ['name']) - vm_ref = NoneV + vm_ref = None for vm in vms: if vm.propSet[0].val == instance_name: vm_ref = vm.obj diff --git a/nova/context.py b/nova/context.py index c113f7ea7..99085ed75 100644 --- a/nova/context.py +++ b/nova/context.py @@ -18,7 +18,6 @@ """RequestContext: context for requests that persist through all of nova.""" -import datetime import random from nova import exception diff --git a/nova/crypto.py b/nova/crypto.py index bdc32482a..8d535f426 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -176,7 +176,8 @@ def revoke_certs_by_project(project_id): def revoke_certs_by_user_and_project(user_id, project_id): """Revoke certs for user in project.""" admin = context.get_admin_context() - for cert in db.certificate_get_all_by_user(admin, user_id, project_id): + for cert in db.certificate_get_all_by_user_and_project(admin, + user_id, project_id): revoke_cert(cert['project_id'], cert['file_name']) diff --git a/nova/db/api.py b/nova/db/api.py index ef8aa1143..e6a46aec9 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -47,6 +47,8 @@ flags.DEFINE_string('instance_name_template', 'instance-%08x', 'Template string to be used to generate instance names') flags.DEFINE_string('volume_name_template', 'volume-%08x', 'Template string to be used to generate instance names') +flags.DEFINE_string('snapshot_name_template', 'snapshot-%08x', + 'Template string to be used to generate snapshot names') IMPL = utils.LazyPluggable(FLAGS['db_backend'], @@ -403,7 +405,7 @@ def instance_create(context, values): def instance_data_get_for_project(context, project_id): - """Get (instance_count, core_count) for project.""" + """Get (instance_count, total_cores, total_ram) for project.""" return IMPL.instance_data_get_for_project(context, project_id) @@ -412,6 +414,16 @@ def instance_destroy(context, instance_id): return IMPL.instance_destroy(context, instance_id) +def instance_stop(context, instance_id): + """Stop the instance or raise if it does not exist.""" + return IMPL.instance_stop(context, instance_id) + + +def instance_get_by_uuid(context, uuid): + """Get an instance or raise if it does not exist.""" + return IMPL.instance_get_by_uuid(context, uuid) + + def instance_get(context, instance_id): """Get an instance or raise if it does not exist.""" return IMPL.instance_get(context, instance_id) @@ -881,6 +893,73 @@ def volume_update(context, volume_id, values): #################### +def snapshot_create(context, values): + """Create a snapshot from the values dictionary.""" + return IMPL.snapshot_create(context, values) + + +def snapshot_destroy(context, snapshot_id): + """Destroy the snapshot or raise if it does not exist.""" + return IMPL.snapshot_destroy(context, snapshot_id) + + +def snapshot_get(context, snapshot_id): + """Get a snapshot or raise if it does not exist.""" + return IMPL.snapshot_get(context, snapshot_id) + + +def snapshot_get_all(context): + """Get all snapshots.""" + return IMPL.snapshot_get_all(context) + + +def snapshot_get_all_by_project(context, project_id): + """Get all snapshots belonging to a project.""" + return IMPL.snapshot_get_all_by_project(context, project_id) + + +def snapshot_update(context, snapshot_id, values): + """Set the given properties on an snapshot and update it. + + Raises NotFound if snapshot does not exist. + + """ + return IMPL.snapshot_update(context, snapshot_id, values) + + +#################### + + +def block_device_mapping_create(context, values): + """Create an entry of block device mapping""" + return IMPL.block_device_mapping_create(context, values) + + +def block_device_mapping_update(context, bdm_id, values): + """Create an entry of block device mapping""" + return IMPL.block_device_mapping_update(context, bdm_id, values) + + +def block_device_mapping_get_all_by_instance(context, instance_id): + """Get all block device mapping belonging to a instance""" + return IMPL.block_device_mapping_get_all_by_instance(context, instance_id) + + +def block_device_mapping_destroy(context, bdm_id): + """Destroy the block device mapping.""" + return IMPL.block_device_mapping_destroy(context, bdm_id) + + +def block_device_mapping_destroy_by_instance_and_volume(context, instance_id, + volume_id): + """Destroy the block device mapping or raise if it does not exist.""" + return IMPL.block_device_mapping_destroy_by_instance_and_volume( + context, instance_id, volume_id) + + +#################### + + def security_group_get_all(context): """Get all security groups.""" return IMPL.security_group_get_all(context) @@ -955,6 +1034,29 @@ def security_group_rule_destroy(context, security_group_rule_id): ################### +def provider_fw_rule_create(context, rule): + """Add a firewall rule at the provider level (all hosts & instances).""" + return IMPL.provider_fw_rule_create(context, rule) + + +def provider_fw_rule_get_all(context): + """Get all provider-level firewall rules.""" + return IMPL.provider_fw_rule_get_all(context) + + +def provider_fw_rule_get_all_by_cidr(context, cidr): + """Get all provider-level firewall rules.""" + return IMPL.provider_fw_rule_get_all_by_cidr(context, cidr) + + +def provider_fw_rule_destroy(context, rule_id): + """Delete a provider firewall rule from the database.""" + return IMPL.provider_fw_rule_destroy(context, rule_id) + + +################### + + def user_get(context, id): """Get user by id.""" return IMPL.user_get(context, id) @@ -1208,3 +1310,32 @@ def instance_metadata_delete(context, instance_id, key): def instance_metadata_update_or_create(context, instance_id, metadata): """Create or update instance metadata.""" IMPL.instance_metadata_update_or_create(context, instance_id, metadata) + + +#################### + + +def agent_build_create(context, values): + """Create a new agent build entry.""" + return IMPL.agent_build_create(context, values) + + +def agent_build_get_by_triple(context, hypervisor, os, architecture): + """Get agent build by hypervisor/OS/architecture triple.""" + return IMPL.agent_build_get_by_triple(context, hypervisor, os, + architecture) + + +def agent_build_get_all(context): + """Get all agent builds.""" + return IMPL.agent_build_get_all(context) + + +def agent_build_destroy(context, agent_update_id): + """Destroy agent build entry.""" + IMPL.agent_build_destroy(context, agent_update_id) + + +def agent_build_update(context, agent_build_id, values): + """Update agent build entry.""" + IMPL.agent_build_update(context, agent_build_id, values) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index b53e81053..b6f5cc717 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -18,8 +18,7 @@ """ Implementation of SQLAlchemy backend. """ - -import datetime +import traceback import warnings from nova import db @@ -61,9 +60,7 @@ def is_user_context(context): def authorize_project_context(context, project_id): - """Ensures that the request context has permission to access the - given project. - """ + """Ensures a request has permission to access the given project.""" if is_user_context(context): if not context.project: raise exception.NotAuthorized() @@ -72,9 +69,7 @@ def authorize_project_context(context, project_id): def authorize_user_context(context, user_id): - """Ensures that the request context has permission to access the - given user. - """ + """Ensures a request has permission to access the given user.""" if is_user_context(context): if not context.user: raise exception.NotAuthorized() @@ -90,9 +85,12 @@ def can_read_deleted(context): def require_admin_context(f): - """Decorator used to indicate that the method requires an - administrator context. + """Decorator to require admin request context. + + The first argument to the wrapped function must be the context. + """ + def wrapper(*args, **kwargs): if not is_admin_context(args[0]): raise exception.AdminRequired() @@ -101,12 +99,19 @@ def require_admin_context(f): def require_context(f): - """Decorator used to indicate that the method requires either - an administrator or normal user context. + """Decorator to require *any* user or admin context. + + This does no authorization for user or project access matching, see + :py:func:`authorize_project_context` and + :py:func:`authorize_user_context`. + + The first argument to the wrapped function must be the context. + """ + def wrapper(*args, **kwargs): if not is_admin_context(args[0]) and not is_user_context(args[0]): - raise exception.AdminRequired() + raise exception.NotAuthorized() return f(*args, **kwargs) return wrapper @@ -674,7 +679,7 @@ def fixed_ip_disassociate_all_by_timeout(_context, host, time): filter_by(allocated=0).\ update({'instance_id': None, 'leased': 0, - 'updated_at': datetime.datetime.utcnow()}, + 'updated_at': utils.utcnow()}, synchronize_session='fetch') return result @@ -738,7 +743,7 @@ def fixed_ip_get_all_by_instance(context, instance_id): filter_by(instance_id=instance_id).\ filter_by(deleted=False) if not rv: - raise exception.NoFloatingIpsFoundForInstance(instance_id=instance_id) + raise exception.NoFixedIpsFoundForInstance(instance_id=instance_id) return rv @@ -771,6 +776,15 @@ def fixed_ip_update(context, address, values): ################### +def _metadata_refs(metadata_dict): + metadata_refs = [] + if metadata_dict: + for k, v in metadata_dict.iteritems(): + metadata_ref = models.InstanceMetadata() + metadata_ref['key'] = k + metadata_ref['value'] = v + metadata_refs.append(metadata_ref) + return metadata_refs @require_context @@ -780,17 +794,11 @@ def instance_create(context, values): context - request context object values - dict containing column values. """ - metadata = values.get('metadata') - metadata_refs = [] - if metadata: - for k, v in metadata.iteritems(): - metadata_ref = models.InstanceMetadata() - metadata_ref['key'] = k - metadata_ref['value'] = v - metadata_refs.append(metadata_ref) - values['metadata'] = metadata_refs + values['metadata'] = _metadata_refs(values.get('metadata')) instance_ref = models.Instance() + instance_ref['uuid'] = str(utils.gen_uuid()) + instance_ref.update(values) session = get_session() @@ -803,12 +811,13 @@ def instance_create(context, values): def instance_data_get_for_project(context, project_id): session = get_session() result = session.query(func.count(models.Instance.id), - func.sum(models.Instance.vcpus)).\ + func.sum(models.Instance.vcpus), + func.sum(models.Instance.memory_mb)).\ filter_by(project_id=project_id).\ filter_by(deleted=False).\ first() # NOTE(vish): convert None to 0 - return (result[0] or 0, result[1] or 0) + return (result[0] or 0, result[1] or 0, result[2] or 0) @require_context @@ -818,52 +827,80 @@ def instance_destroy(context, instance_id): session.query(models.Instance).\ filter_by(id=instance_id).\ update({'deleted': True, - 'deleted_at': datetime.datetime.utcnow(), + 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupInstanceAssociation).\ filter_by(instance_id=instance_id).\ update({'deleted': True, - 'deleted_at': datetime.datetime.utcnow(), + 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) session.query(models.InstanceMetadata).\ filter_by(instance_id=instance_id).\ update({'deleted': True, - 'deleted_at': datetime.datetime.utcnow(), + 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) @require_context +def instance_stop(context, instance_id): + session = get_session() + with session.begin(): + from nova.compute import power_state + session.query(models.Instance).\ + filter_by(id=instance_id).\ + update({'host': None, + 'state': power_state.SHUTOFF, + 'state_description': 'stopped', + 'updated_at': literal_column('updated_at')}) + session.query(models.SecurityGroupInstanceAssociation).\ + filter_by(instance_id=instance_id).\ + update({'updated_at': literal_column('updated_at')}) + session.query(models.InstanceMetadata).\ + filter_by(instance_id=instance_id).\ + update({'updated_at': literal_column('updated_at')}) + + +@require_context +def instance_get_by_uuid(context, uuid, session=None): + partial = _build_instance_get(context, session=session) + result = partial.filter_by(uuid=uuid) + result = result.first() + if not result: + # FIXME(sirp): it would be nice if InstanceNotFound would accept a + # uuid parameter as well + raise exception.InstanceNotFound(instance_id=uuid) + return result + + +@require_context def instance_get(context, instance_id, session=None): + partial = _build_instance_get(context, session=session) + result = partial.filter_by(id=instance_id) + result = result.first() + if not result: + raise exception.InstanceNotFound(instance_id=instance_id) + return result + + +@require_context +def _build_instance_get(context, session=None): if not session: session = get_session() - result = None + + partial = session.query(models.Instance).\ + options(joinedload_all('fixed_ip.floating_ips')).\ + options(joinedload_all('security_groups.rules')).\ + options(joinedload('volumes')).\ + options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ + options(joinedload('instance_type')) if is_admin_context(context): - result = session.query(models.Instance).\ - options(joinedload_all('fixed_ip.floating_ips')).\ - options(joinedload_all('security_groups.rules')).\ - options(joinedload('volumes')).\ - options(joinedload_all('fixed_ip.network')).\ - options(joinedload('metadata')).\ - options(joinedload('instance_type')).\ - filter_by(id=instance_id).\ - filter_by(deleted=can_read_deleted(context)).\ - first() + partial = partial.filter_by(deleted=can_read_deleted(context)) elif is_user_context(context): - result = session.query(models.Instance).\ - options(joinedload_all('fixed_ip.floating_ips')).\ - options(joinedload_all('security_groups.rules')).\ - options(joinedload('volumes')).\ - options(joinedload('metadata')).\ - options(joinedload('instance_type')).\ - filter_by(project_id=context.project_id).\ - filter_by(id=instance_id).\ - filter_by(deleted=False).\ - first() - if not result: - raise exception.InstanceNotFound(instance_id=instance_id) - - return result + partial = partial.filter_by(project_id=context.project_id).\ + filter_by(deleted=False) + return partial @require_admin_context @@ -900,6 +937,7 @@ def instance_get_all_by_host(context, host): options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload('security_groups')).\ options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ options(joinedload('instance_type')).\ filter_by(host=host).\ filter_by(deleted=can_read_deleted(context)).\ @@ -915,6 +953,7 @@ def instance_get_all_by_project(context, project_id): options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload('security_groups')).\ options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ options(joinedload('instance_type')).\ filter_by(project_id=project_id).\ filter_by(deleted=can_read_deleted(context)).\ @@ -930,6 +969,7 @@ def instance_get_all_by_reservation(context, reservation_id): options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload('security_groups')).\ options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ options(joinedload('instance_type')).\ filter_by(reservation_id=reservation_id).\ filter_by(deleted=can_read_deleted(context)).\ @@ -939,6 +979,7 @@ def instance_get_all_by_reservation(context, reservation_id): options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload('security_groups')).\ options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ options(joinedload('instance_type')).\ filter_by(project_id=context.project_id).\ filter_by(reservation_id=reservation_id).\ @@ -952,9 +993,11 @@ def instance_get_project_vpn(context, project_id): return session.query(models.Instance).\ options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload('security_groups')).\ + options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ options(joinedload('instance_type')).\ filter_by(project_id=project_id).\ - filter_by(image_id=str(FLAGS.vpn_image_id)).\ + filter_by(image_ref=str(FLAGS.vpn_image_id)).\ filter_by(deleted=can_read_deleted(context)).\ first() @@ -1009,6 +1052,11 @@ def instance_set_state(context, instance_id, state, description=None): @require_context def instance_update(context, instance_id, values): session = get_session() + metadata = values.get('metadata') + if metadata is not None: + instance_metadata_delete_all(context, instance_id) + instance_metadata_update_or_create(context, instance_id, + values.pop('metadata')) with session.begin(): instance_ref = instance_get(context, instance_id, session=session) instance_ref.update(values) @@ -1115,8 +1163,8 @@ def key_pair_destroy_all_by_user(context, user_id): with session.begin(): session.query(models.KeyPair).\ filter_by(user_id=user_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), + update({'deleted': True, + 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) @@ -1499,7 +1547,7 @@ def auth_token_create(_context, token): ################### -@require_admin_context +@require_context def quota_get(context, project_id, resource, session=None): if not session: session = get_session() @@ -1513,7 +1561,7 @@ def quota_get(context, project_id, resource, session=None): return result -@require_admin_context +@require_context def quota_get_all_by_project(context, project_id): session = get_session() result = {'project_id': project_id} @@ -1647,8 +1695,8 @@ def volume_destroy(context, volume_id): with session.begin(): session.query(models.Volume).\ filter_by(id=volume_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), + update({'deleted': True, + 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) session.query(models.ExportDevice).\ filter_by(volume_id=volume_id).\ @@ -1789,6 +1837,142 @@ def volume_update(context, volume_id, values): @require_context +def snapshot_create(context, values): + snapshot_ref = models.Snapshot() + snapshot_ref.update(values) + + session = get_session() + with session.begin(): + snapshot_ref.save(session=session) + return snapshot_ref + + +@require_admin_context +def snapshot_destroy(context, snapshot_id): + session = get_session() + with session.begin(): + session.query(models.Snapshot).\ + filter_by(id=snapshot_id).\ + update({'deleted': True, + 'deleted_at': utils.utcnow(), + 'updated_at': literal_column('updated_at')}) + + +@require_context +def snapshot_get(context, snapshot_id, session=None): + if not session: + session = get_session() + result = None + + if is_admin_context(context): + result = session.query(models.Snapshot).\ + filter_by(id=snapshot_id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() + elif is_user_context(context): + result = session.query(models.Snapshot).\ + filter_by(project_id=context.project_id).\ + filter_by(id=snapshot_id).\ + filter_by(deleted=False).\ + first() + if not result: + raise exception.SnapshotNotFound(snapshot_id=snapshot_id) + + return result + + +@require_admin_context +def snapshot_get_all(context): + session = get_session() + return session.query(models.Snapshot).\ + filter_by(deleted=can_read_deleted(context)).\ + all() + + +@require_context +def snapshot_get_all_by_project(context, project_id): + authorize_project_context(context, project_id) + + session = get_session() + return session.query(models.Snapshot).\ + filter_by(project_id=project_id).\ + filter_by(deleted=can_read_deleted(context)).\ + all() + + +@require_context +def snapshot_update(context, snapshot_id, values): + session = get_session() + with session.begin(): + snapshot_ref = snapshot_get(context, snapshot_id, session=session) + snapshot_ref.update(values) + snapshot_ref.save(session=session) + + +################### + + +@require_context +def block_device_mapping_create(context, values): + bdm_ref = models.BlockDeviceMapping() + bdm_ref.update(values) + + session = get_session() + with session.begin(): + bdm_ref.save(session=session) + + +@require_context +def block_device_mapping_update(context, bdm_id, values): + session = get_session() + with session.begin(): + session.query(models.BlockDeviceMapping).\ + filter_by(id=bdm_id).\ + filter_by(deleted=False).\ + update(values) + + +@require_context +def block_device_mapping_get_all_by_instance(context, instance_id): + session = get_session() + result = session.query(models.BlockDeviceMapping).\ + filter_by(instance_id=instance_id).\ + filter_by(deleted=False).\ + all() + if not result: + return [] + return result + + +@require_context +def block_device_mapping_destroy(context, bdm_id): + session = get_session() + with session.begin(): + session.query(models.BlockDeviceMapping).\ + filter_by(id=bdm_id).\ + update({'deleted': True, + 'deleted_at': utils.utcnow(), + 'updated_at': literal_column('updated_at')}) + + +@require_context +def block_device_mapping_destroy_by_instance_and_volume(context, instance_id, + volume_id): + session = get_session() + with session.begin(): + session.query(models.BlockDeviceMapping).\ + filter_by(instance_id=instance_id).\ + filter_by(volume_id=volume_id).\ + filter_by(deleted=False).\ + update({'deleted': True, + 'deleted_at': utils.utcnow(), + 'updated_at': literal_column('updated_at')}) + + +################### + + +@require_context def security_group_get_all(context): session = get_session() return session.query(models.SecurityGroup).\ @@ -1884,18 +2068,18 @@ def security_group_destroy(context, security_group_id): with session.begin(): session.query(models.SecurityGroup).\ filter_by(id=security_group_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), + update({'deleted': True, + 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupInstanceAssociation).\ filter_by(security_group_id=security_group_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), + update({'deleted': True, + 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupIngressRule).\ filter_by(group_id=security_group_id).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), + update({'deleted': True, + 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) @@ -1905,12 +2089,12 @@ def security_group_destroy_all(context, session=None): session = get_session() with session.begin(): session.query(models.SecurityGroup).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), + update({'deleted': True, + 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) session.query(models.SecurityGroupIngressRule).\ - update({'deleted': 1, - 'deleted_at': datetime.datetime.utcnow(), + update({'deleted': True, + 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) @@ -1996,6 +2180,46 @@ def security_group_rule_destroy(context, security_group_rule_id): ################### + +@require_admin_context +def provider_fw_rule_create(context, rule): + fw_rule_ref = models.ProviderFirewallRule() + fw_rule_ref.update(rule) + fw_rule_ref.save() + return fw_rule_ref + + +@require_admin_context +def provider_fw_rule_get_all(context): + session = get_session() + return session.query(models.ProviderFirewallRule).\ + filter_by(deleted=can_read_deleted(context)).\ + all() + + +@require_admin_context +def provider_fw_rule_get_all_by_cidr(context, cidr): + session = get_session() + return session.query(models.ProviderFirewallRule).\ + filter_by(deleted=can_read_deleted(context)).\ + filter_by(cidr=cidr).\ + all() + + +@require_admin_context +def provider_fw_rule_destroy(context, rule_id): + session = get_session() + with session.begin(): + session.query(models.ProviderFirewallRule).\ + filter_by(id=rule_id).\ + update({'deleted': True, + 'deleted_at': utils.utcnow(), + 'updated_at': literal_column('updated_at')}) + + +################### + + @require_admin_context def user_get(context, id, session=None): if not session: @@ -2521,7 +2745,17 @@ def zone_get_all(context): #################### + +def require_instance_exists(func): + def new_func(context, instance_id, *args, **kwargs): + db.api.instance_get(context, instance_id) + return func(context, instance_id, *args, **kwargs) + new_func.__name__ = func.__name__ + return new_func + + @require_context +@require_instance_exists def instance_metadata_get(context, instance_id): session = get_session() @@ -2537,6 +2771,7 @@ def instance_metadata_get(context, instance_id): @require_context +@require_instance_exists def instance_metadata_delete(context, instance_id, key): session = get_session() session.query(models.InstanceMetadata).\ @@ -2544,11 +2779,24 @@ def instance_metadata_delete(context, instance_id, key): filter_by(key=key).\ filter_by(deleted=False).\ update({'deleted': True, - 'deleted_at': datetime.datetime.utcnow(), + 'deleted_at': utils.utcnow(), + 'updated_at': literal_column('updated_at')}) + + +@require_context +@require_instance_exists +def instance_metadata_delete_all(context, instance_id): + session = get_session() + session.query(models.InstanceMetadata).\ + filter_by(instance_id=instance_id).\ + filter_by(deleted=False).\ + update({'deleted': True, + 'deleted_at': utils.utcnow(), 'updated_at': literal_column('updated_at')}) @require_context +@require_instance_exists def instance_metadata_get_item(context, instance_id, key): session = get_session() @@ -2565,8 +2813,12 @@ def instance_metadata_get_item(context, instance_id, key): @require_context +@require_instance_exists def instance_metadata_update_or_create(context, instance_id, metadata): session = get_session() + + original_metadata = instance_metadata_get(context, instance_id) + meta_ref = None for key, value in metadata.iteritems(): try: @@ -2576,6 +2828,58 @@ def instance_metadata_update_or_create(context, instance_id, metadata): meta_ref = models.InstanceMetadata() meta_ref.update({"key": key, "value": value, "instance_id": instance_id, - "deleted": 0}) + "deleted": False}) meta_ref.save(session=session) + return metadata + + +@require_admin_context +def agent_build_create(context, values): + agent_build_ref = models.AgentBuild() + agent_build_ref.update(values) + agent_build_ref.save() + return agent_build_ref + + +@require_admin_context +def agent_build_get_by_triple(context, hypervisor, os, architecture, + session=None): + if not session: + session = get_session() + return session.query(models.AgentBuild).\ + filter_by(hypervisor=hypervisor).\ + filter_by(os=os).\ + filter_by(architecture=architecture).\ + filter_by(deleted=False).\ + first() + + +@require_admin_context +def agent_build_get_all(context): + session = get_session() + return session.query(models.AgentBuild).\ + filter_by(deleted=False).\ + all() + + +@require_admin_context +def agent_build_destroy(context, agent_build_id): + session = get_session() + with session.begin(): + session.query(models.AgentBuild).\ + filter_by(id=agent_build_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) + + +@require_admin_context +def agent_build_update(context, agent_build_id, values): + session = get_session() + with session.begin(): + agent_build_ref = session.query(models.AgentBuild).\ + filter_by(id=agent_build_id). \ + first() + agent_build_ref.update(values) + agent_build_ref.save(session=session) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/016_make_quotas_key_and_value.py b/nova/db/sqlalchemy/migrate_repo/versions/016_make_quotas_key_and_value.py index a2d8192ca..a4fe3e482 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/016_make_quotas_key_and_value.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/016_make_quotas_key_and_value.py @@ -17,7 +17,7 @@ from sqlalchemy import Boolean, Column, DateTime, Integer from sqlalchemy import MetaData, String, Table -import datetime +from nova import utils meta = MetaData() @@ -35,9 +35,9 @@ def old_style_quotas_table(name): return Table(name, meta, Column('id', Integer(), primary_key=True), Column('created_at', DateTime(), - default=datetime.datetime.utcnow), + default=utils.utcnow), Column('updated_at', DateTime(), - onupdate=datetime.datetime.utcnow), + onupdate=utils.utcnow), Column('deleted_at', DateTime()), Column('deleted', Boolean(), default=False), Column('project_id', @@ -57,9 +57,9 @@ def new_style_quotas_table(name): return Table(name, meta, Column('id', Integer(), primary_key=True), Column('created_at', DateTime(), - default=datetime.datetime.utcnow), + default=utils.utcnow), Column('updated_at', DateTime(), - onupdate=datetime.datetime.utcnow), + onupdate=utils.utcnow), Column('deleted_at', DateTime()), Column('deleted', Boolean(), default=False), Column('project_id', @@ -160,7 +160,7 @@ def convert_backward(migrate_engine, old_quotas, new_quotas): 'project_id': quota.project_id, 'created_at': quota.created_at, 'updated_at': quota.updated_at, - quota.resource: quota.hard_limit + quota.resource: quota.hard_limit, } else: quotas[quota.project_id]['created_at'] = earliest( diff --git a/nova/db/sqlalchemy/migrate_repo/versions/018_rename_server_management_url.py b/nova/db/sqlalchemy/migrate_repo/versions/018_rename_server_management_url.py index a169afb40..73c76f666 100644 --- a/nova/db/sqlalchemy/migrate_repo/versions/018_rename_server_management_url.py +++ b/nova/db/sqlalchemy/migrate_repo/versions/018_rename_server_management_url.py @@ -14,23 +14,10 @@ # License for the specific language governing permissions and limitations # under the License. -from sqlalchemy import Column, Integer, MetaData, String, Table -#from nova import log as logging +from sqlalchemy import MetaData, Table meta = MetaData() -c_manageent = Column('server_manageent_url', - String(length=255, convert_unicode=False, - assert_unicode=None, unicode_error=None, - _warn_on_bytestring=False), - nullable=True) - -c_management = Column('server_management_url', - String(length=255, convert_unicode=False, - assert_unicode=None, unicode_error=None, - _warn_on_bytestring=False), - nullable=True) - def upgrade(migrate_engine): # Upgrade operations go here. Don't create your own engine; @@ -40,11 +27,8 @@ def upgrade(migrate_engine): tokens = Table('auth_tokens', meta, autoload=True, autoload_with=migrate_engine) - tokens.create_column(c_management) - migrate_engine.execute(tokens.update() - .values(server_management_url=tokens.c.server_manageent_url)) - - tokens.c.server_manageent_url.drop() + c_manageent = tokens.c.server_manageent_url + c_manageent.alter(name='server_management_url') def downgrade(migrate_engine): @@ -53,8 +37,5 @@ def downgrade(migrate_engine): tokens = Table('auth_tokens', meta, autoload=True, autoload_with=migrate_engine) - tokens.create_column(c_manageent) - migrate_engine.execute(tokens.update() - .values(server_manageent_url=tokens.c.server_management_url)) - - tokens.c.server_management_url.drop() + c_management = tokens.c.server_management_url + c_management.alter(name='server_manageent_url') diff --git a/nova/db/sqlalchemy/migrate_repo/versions/019_add_volume_snapshot_support.py b/nova/db/sqlalchemy/migrate_repo/versions/019_add_volume_snapshot_support.py new file mode 100644 index 000000000..f16d6db56 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/019_add_volume_snapshot_support.py @@ -0,0 +1,70 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 MORITA Kazutaka. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Column, Table, MetaData +from sqlalchemy import Integer, DateTime, Boolean, String + +from nova import log as logging + +meta = MetaData() + +snapshots = Table('snapshots', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('volume_id', Integer(), nullable=False), + Column('user_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('project_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('status', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('progress', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('volume_size', Integer()), + Column('scheduled_at', DateTime(timezone=False)), + Column('display_name', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('display_description', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False))) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + try: + snapshots.create() + except Exception: + logging.info(repr(snapshots)) + logging.exception('Exception while creating table') + meta.drop_all(tables=[snapshots]) + raise + + +def downgrade(migrate_engine): + # Operations to reverse the above upgrade go here. + snapshots.drop() diff --git a/nova/db/sqlalchemy/migrate_repo/versions/020_add_snapshot_id_to_volumes.py b/nova/db/sqlalchemy/migrate_repo/versions/020_add_snapshot_id_to_volumes.py new file mode 100644 index 000000000..10bd9d5c9 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/020_add_snapshot_id_to_volumes.py @@ -0,0 +1,47 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 MORITA Kazutaka. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Column, Table, MetaData, Integer + +from nova import log as logging + + +meta = MetaData() + + +# Table stub-definitions +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +# +volumes = Table('volumes', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +# +# New Column +# + +snapshot_id = Column('snapshot_id', Integer()) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + # Add columns to existing tables + volumes.create_column(snapshot_id) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/021_rename_image_ids.py b/nova/db/sqlalchemy/migrate_repo/versions/021_rename_image_ids.py new file mode 100644 index 000000000..73a5e8477 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/021_rename_image_ids.py @@ -0,0 +1,40 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Column, Integer, MetaData, String, Table + + +meta = MetaData() + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + instances = Table('instances', meta, autoload=True, + autoload_with=migrate_engine) + + image_id_column = instances.c.image_id + image_id_column.alter(name='image_ref') + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + instances = Table('instances', meta, autoload=True, + autoload_with=migrate_engine) + + image_ref_column = instances.c.image_ref + image_ref_column.alter(name='image_id') diff --git a/nova/db/sqlalchemy/migrate_repo/versions/022_set_engine_mysql_innodb.py b/nova/db/sqlalchemy/migrate_repo/versions/022_set_engine_mysql_innodb.py new file mode 100644 index 000000000..6e590479f --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/022_set_engine_mysql_innodb.py @@ -0,0 +1,65 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import MetaData, Table + +meta = MetaData() + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + if migrate_engine.name == "mysql": + migrate_engine.execute("ALTER TABLE auth_tokens Engine=InnoDB") + migrate_engine.execute("ALTER TABLE certificates Engine=InnoDB") + migrate_engine.execute("ALTER TABLE compute_nodes Engine=InnoDB") + migrate_engine.execute("ALTER TABLE console_pools Engine=InnoDB") + migrate_engine.execute("ALTER TABLE consoles Engine=InnoDB") + migrate_engine.execute("ALTER TABLE export_devices Engine=InnoDB") + migrate_engine.execute("ALTER TABLE fixed_ips Engine=InnoDB") + migrate_engine.execute("ALTER TABLE floating_ips Engine=InnoDB") + migrate_engine.execute("ALTER TABLE instance_actions Engine=InnoDB") + migrate_engine.execute("ALTER TABLE instance_metadata Engine=InnoDB") + migrate_engine.execute("ALTER TABLE instance_types Engine=InnoDB") + migrate_engine.execute("ALTER TABLE instances Engine=InnoDB") + migrate_engine.execute("ALTER TABLE iscsi_targets Engine=InnoDB") + migrate_engine.execute("ALTER TABLE key_pairs Engine=InnoDB") + migrate_engine.execute("ALTER TABLE migrate_version Engine=InnoDB") + migrate_engine.execute("ALTER TABLE migrations Engine=InnoDB") + migrate_engine.execute("ALTER TABLE networks Engine=InnoDB") + migrate_engine.execute("ALTER TABLE projects Engine=InnoDB") + migrate_engine.execute("ALTER TABLE quotas Engine=InnoDB") + migrate_engine.execute( + "ALTER TABLE security_group_instance_association Engine=InnoDB") + migrate_engine.execute( + "ALTER TABLE security_group_rules Engine=InnoDB") + migrate_engine.execute("ALTER TABLE security_groups Engine=InnoDB") + migrate_engine.execute("ALTER TABLE services Engine=InnoDB") + migrate_engine.execute( + "ALTER TABLE user_project_association Engine=InnoDB") + migrate_engine.execute( + "ALTER TABLE user_project_role_association Engine=InnoDB") + migrate_engine.execute( + "ALTER TABLE user_role_association Engine=InnoDB") + migrate_engine.execute("ALTER TABLE users Engine=InnoDB") + migrate_engine.execute("ALTER TABLE volumes Engine=InnoDB") + migrate_engine.execute("ALTER TABLE zones Engine=InnoDB") + migrate_engine.execute("ALTER TABLE snapshots Engine=InnoDB") + + +def downgrade(migrate_engine): + meta.bind = migrate_engine diff --git a/nova/db/sqlalchemy/migrate_repo/versions/023_add_vm_mode_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/023_add_vm_mode_to_instances.py new file mode 100644 index 000000000..0c587f569 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/023_add_vm_mode_to_instances.py @@ -0,0 +1,45 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Column, Integer, MetaData, String, Table + +meta = MetaData() + +instances_vm_mode = Column('vm_mode', + String(length=255, convert_unicode=False, + assert_unicode=None, unicode_error=None, + _warn_on_bytestring=False), + nullable=True) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + instances = Table('instances', meta, autoload=True, + autoload_with=migrate_engine) + + instances.create_column(instances_vm_mode) + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + + instances = Table('instances', meta, autoload=True, + autoload_with=migrate_engine) + + instances.drop_column('vm_mode') diff --git a/nova/db/sqlalchemy/migrate_repo/versions/024_add_block_device_mapping.py b/nova/db/sqlalchemy/migrate_repo/versions/024_add_block_device_mapping.py new file mode 100644 index 000000000..6e9b806cb --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/024_add_block_device_mapping.py @@ -0,0 +1,87 @@ +# Copyright 2011 OpenStack LLC. +# Copyright 2011 Isaku Yamahata +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import MetaData, Table, Column +from sqlalchemy import DateTime, Boolean, Integer, String +from sqlalchemy import ForeignKey +from nova import log as logging + +meta = MetaData() + +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +volumes = Table('volumes', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +snapshots = Table('snapshots', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +block_device_mapping = Table('block_device_mapping', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, autoincrement=True), + Column('instance_id', + Integer(), + ForeignKey('instances.id'), + nullable=False), + Column('device_name', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=False), + Column('delete_on_termination', + Boolean(create_constraint=True, name=None), + default=False), + Column('virtual_name', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=True), + Column('snapshot_id', + Integer(), + ForeignKey('snapshots.id'), + nullable=True), + Column('volume_id', Integer(), ForeignKey('volumes.id'), + nullable=True), + Column('volume_size', Integer(), nullable=True), + Column('no_device', + Boolean(create_constraint=True, name=None), + nullable=True), + ) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + try: + block_device_mapping.create() + except Exception: + logging.info(repr(block_device_mapping)) + logging.exception('Exception while creating table') + meta.drop_all(tables=[block_device_mapping]) + raise + + +def downgrade(migrate_engine): + # Operations to reverse the above upgrade go here. + block_device_mapping.drop() diff --git a/nova/db/sqlalchemy/migrate_repo/versions/025_add_uuid_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/025_add_uuid_to_instances.py new file mode 100644 index 000000000..27f30d536 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/025_add_uuid_to_instances.py @@ -0,0 +1,43 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Column, Integer, MetaData, String, Table + +from nova import utils + + +meta = MetaData() + +instances = Table("instances", meta, + Column("id", Integer(), primary_key=True, nullable=False)) +uuid_column = Column("uuid", String(36)) + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + instances.create_column(uuid_column) + + rows = migrate_engine.execute(instances.select()) + for row in rows: + instance_uuid = str(utils.gen_uuid()) + migrate_engine.execute(instances.update()\ + .where(instances.c.id == row[0])\ + .values(uuid=instance_uuid)) + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + instances.drop_column(uuid_column) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/026_add_agent_table.py b/nova/db/sqlalchemy/migrate_repo/versions/026_add_agent_table.py new file mode 100644 index 000000000..640e96138 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/026_add_agent_table.py @@ -0,0 +1,73 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Boolean, Column, DateTime, Integer +from sqlalchemy import MetaData, String, Table +from nova import log as logging + +meta = MetaData() + +# +# New Tables +# +builds = Table('agent_builds', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('hypervisor', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('os', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('architecture', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('version', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('url', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('md5hash', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + + +# +# New Column +# + +architecture = Column('architecture', String(length=255)) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (builds, ): + try: + table.create() + except Exception: + logging.info(repr(table)) + + instances = Table('instances', meta, autoload=True, + autoload_with=migrate_engine) + + # Add columns to existing tables + instances.create_column(architecture) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/027_add_provider_firewall_rules.py b/nova/db/sqlalchemy/migrate_repo/versions/027_add_provider_firewall_rules.py new file mode 100644 index 000000000..5aa30f7a8 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/027_add_provider_firewall_rules.py @@ -0,0 +1,75 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + + +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +services = Table('services', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +networks = Table('networks', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +# +# New Tables +# +provider_fw_rules = Table('provider_fw_rules', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('protocol', + String(length=5, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('from_port', Integer()), + Column('to_port', Integer()), + Column('cidr', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)) + ) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (provider_fw_rules,): + try: + table.create() + except Exception: + logging.info(repr(table)) + logging.exception('Exception while creating table') + raise diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 1215448f8..a5e2b1008 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -19,8 +19,6 @@ SQLAlchemy models for nova data. """ -import datetime - from sqlalchemy.orm import relationship, backref, object_mapper from sqlalchemy import Column, Integer, String, schema from sqlalchemy import ForeignKey, DateTime, Boolean, Text @@ -33,6 +31,7 @@ from nova.db.sqlalchemy.session import get_session from nova import auth from nova import exception from nova import flags +from nova import utils FLAGS = flags.FLAGS @@ -43,10 +42,11 @@ class NovaBase(object): """Base class for Nova Models.""" __table_args__ = {'mysql_engine': 'InnoDB'} __table_initialized__ = False - created_at = Column(DateTime, default=datetime.datetime.utcnow) - updated_at = Column(DateTime, onupdate=datetime.datetime.utcnow) + created_at = Column(DateTime, default=utils.utcnow) + updated_at = Column(DateTime, onupdate=utils.utcnow) deleted_at = Column(DateTime) deleted = Column(Boolean, default=False) + metadata = None def save(self, session=None): """Save this object.""" @@ -64,7 +64,7 @@ class NovaBase(object): def delete(self, session=None): """Delete this object.""" self.deleted = True - self.deleted_at = datetime.datetime.utcnow() + self.deleted_at = utils.utcnow() self.save(session=session) def __setitem__(self, key, value): @@ -184,11 +184,11 @@ class Instance(BASE, NovaBase): def project(self): return auth.manager.AuthManager().get_project(self.project_id) - image_id = Column(String(255)) + image_ref = Column(String(255)) kernel_id = Column(String(255)) ramdisk_id = Column(String(255)) -# image_id = Column(Integer, ForeignKey('images.id'), nullable=True) +# image_ref = Column(Integer, ForeignKey('images.id'), nullable=True) # kernel_id = Column(Integer, ForeignKey('images.id'), nullable=True) # ramdisk_id = Column(Integer, ForeignKey('images.id'), nullable=True) # ramdisk = relationship(Ramdisk, backref=backref('instances', order_by=id)) @@ -232,6 +232,9 @@ class Instance(BASE, NovaBase): locked = Column(Boolean) os_type = Column(String(255)) + architecture = Column(String(255)) + vm_mode = Column(String(255)) + uuid = Column(String(36)) # TODO(vish): see Ewan's email about state improvements, probably # should be in a driver base class or some such @@ -287,6 +290,8 @@ class Volume(BASE, NovaBase): user_id = Column(String(255)) project_id = Column(String(255)) + snapshot_id = Column(String(255)) + host = Column(String(255)) # , ForeignKey('hosts.id')) size = Column(Integer) availability_zone = Column(String(255)) # TODO(vish): foreign key? @@ -329,6 +334,70 @@ class Quota(BASE, NovaBase): hard_limit = Column(Integer, nullable=True) +class Snapshot(BASE, NovaBase): + """Represents a block storage device that can be attached to a vm.""" + __tablename__ = 'snapshots' + id = Column(Integer, primary_key=True, autoincrement=True) + + @property + def name(self): + return FLAGS.snapshot_name_template % self.id + + @property + def volume_name(self): + return FLAGS.volume_name_template % self.volume_id + + user_id = Column(String(255)) + project_id = Column(String(255)) + + volume_id = Column(Integer) + status = Column(String(255)) + progress = Column(String(255)) + volume_size = Column(Integer) + + display_name = Column(String(255)) + display_description = Column(String(255)) + + +class BlockDeviceMapping(BASE, NovaBase): + """Represents block device mapping that is defined by EC2""" + __tablename__ = "block_device_mapping" + id = Column(Integer, primary_key=True, autoincrement=True) + + instance_id = Column(Integer, ForeignKey('instances.id'), nullable=False) + instance = relationship(Instance, + backref=backref('balock_device_mapping'), + foreign_keys=instance_id, + primaryjoin='and_(BlockDeviceMapping.instance_id==' + 'Instance.id,' + 'BlockDeviceMapping.deleted==' + 'False)') + device_name = Column(String(255), nullable=False) + + # default=False for compatibility of the existing code. + # With EC2 API, + # default True for ami specified device. + # default False for created with other timing. + delete_on_termination = Column(Boolean, default=False) + + # for ephemeral device + virtual_name = Column(String(255), nullable=True) + + # for snapshot or volume + snapshot_id = Column(Integer, ForeignKey('snapshots.id'), nullable=True) + # outer join + snapshot = relationship(Snapshot, + foreign_keys=snapshot_id) + + volume_id = Column(Integer, ForeignKey('volumes.id'), nullable=True) + volume = relationship(Volume, + foreign_keys=volume_id) + volume_size = Column(Integer, nullable=True) + + # for no device to suppress devices. + no_device = Column(Boolean, nullable=True) + + class ExportDevice(BASE, NovaBase): """Represates a shelf and blade that a volume can be exported on.""" __tablename__ = 'export_devices' @@ -424,6 +493,17 @@ class SecurityGroupIngressRule(BASE, NovaBase): group_id = Column(Integer, ForeignKey('security_groups.id')) +class ProviderFirewallRule(BASE, NovaBase): + """Represents a rule in a security group.""" + __tablename__ = 'provider_fw_rules' + id = Column(Integer, primary_key=True) + + protocol = Column(String(5)) # "tcp", "udp", or "icmp" + from_port = Column(Integer) + to_port = Column(Integer) + cidr = Column(String(255)) + + class KeyPair(BASE, NovaBase): """Represents a public key pair for ssh.""" __tablename__ = 'key_pairs' @@ -645,6 +725,18 @@ class Zone(BASE, NovaBase): password = Column(String(255)) +class AgentBuild(BASE, NovaBase): + """Represents an agent build.""" + __tablename__ = 'agent_builds' + id = Column(Integer, primary_key=True) + hypervisor = Column(String(255)) + os = Column(String(255)) + architecture = Column(String(255)) + version = Column(String(255)) + url = Column(String(255)) + md5hash = Column(String(255)) + + def register_models(): """Register Models and create metadata. @@ -658,7 +750,7 @@ def register_models(): Network, SecurityGroup, SecurityGroupIngressRule, SecurityGroupInstanceAssociation, AuthToken, User, Project, Certificate, ConsolePool, Console, Zone, - InstanceMetadata, Migration) + AgentBuild, InstanceMetadata, Migration) engine = create_engine(FLAGS.sql_connection, echo=False) for model in models: model.metadata.create_all(engine) diff --git a/nova/exception.py b/nova/exception.py index 56c20d111..f3a452228 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -65,7 +65,7 @@ class BuildInProgress(Error): class DBError(Error): """Wraps an implementation specific exception.""" - def __init__(self, inner_exception): + def __init__(self, inner_exception=None): self.inner_exception = inner_exception super(DBError, self).__init__(str(inner_exception)) @@ -122,7 +122,7 @@ class NotAuthorized(NovaException): message = _("Not authorized.") def __init__(self, *args, **kwargs): - super(NotFound, self).__init__(**kwargs) + super(NotAuthorized, self).__init__(**kwargs) class AdminRequired(NotAuthorized): @@ -271,6 +271,14 @@ class VolumeNotFoundForInstance(VolumeNotFound): message = _("Volume not found for instance %(instance_id)s.") +class SnapshotNotFound(NotFound): + message = _("Snapshot %(snapshot_id)s could not be found.") + + +class VolumeIsBusy(Error): + message = _("deleting volume %(volume_name)s that has snapshot") + + class ExportDeviceNotFoundForVolume(NotFound): message = _("No export device found for volume %(volume_id)s.") @@ -283,6 +291,15 @@ class DiskNotFound(NotFound): message = _("No disk at %(location)s") +class InvalidImageRef(Invalid): + message = _("Invalid image href %(image_href)s.") + + +class ListingImageRefsNotSupported(Invalid): + message = _("Some images have been stored via hrefs." + + " This version of the api does not support displaying image hrefs.") + + class ImageNotFound(NotFound): message = _("Image %(image_id)s could not be found.") @@ -359,6 +376,10 @@ class NoFloatingIpsDefinedForInstance(NoFloatingIpsDefined): message = _("Zero floating ips defined for instance %(instance_id)s.") +class NoMoreFloatingIps(NotFound): + message = _("Zero floating ips available.") + + class KeypairNotFound(NotFound): message = _("Keypair %(keypair_name)s not found for user %(user_id)s") @@ -465,11 +486,19 @@ class ZoneNotFound(NotFound): message = _("Zone %(zone_id)s could not be found.") -class SchedulerHostFilterDriverNotFound(NotFound): - message = _("Scheduler Host Filter Driver %(driver_name)s could" +class SchedulerHostFilterNotFound(NotFound): + message = _("Scheduler Host Filter %(filter_name)s could not be found.") + + +class SchedulerCostFunctionNotFound(NotFound): + message = _("Scheduler cost function %(cost_fn_str)s could" " not be found.") +class SchedulerWeightFlagNotFound(NotFound): + message = _("Scheduler weight flag not found: %(flag_name)s") + + class InstanceMetadataNotFound(NotFound): message = _("Instance %(instance_id)s has no metadata with " "key %(metadata_key)s.") @@ -556,3 +585,7 @@ class InstanceExists(Duplicate): class MigrationError(NovaException): message = _("Migration error") + ": %(reason)s" + + +class MalformedRequestBody(NovaException): + message = _("Malformed message body: %(reason)s") diff --git a/nova/fakerabbit.py b/nova/fakerabbit.py index a7dee8caf..e7e9dab77 100644 --- a/nova/fakerabbit.py +++ b/nova/fakerabbit.py @@ -31,6 +31,7 @@ LOG = logging.getLogger("nova.fakerabbit") EXCHANGES = {} QUEUES = {} +CONSUMERS = {} class Message(base.BaseMessage): @@ -96,17 +97,29 @@ class Backend(base.BaseBackend): ' key %(routing_key)s') % locals()) EXCHANGES[exchange].bind(QUEUES[queue].push, routing_key) - def declare_consumer(self, queue, callback, *args, **kwargs): - self.current_queue = queue - self.current_callback = callback + def declare_consumer(self, queue, callback, consumer_tag, *args, **kwargs): + global CONSUMERS + LOG.debug("Adding consumer %s", consumer_tag) + CONSUMERS[consumer_tag] = (queue, callback) + + def cancel(self, consumer_tag): + global CONSUMERS + LOG.debug("Removing consumer %s", consumer_tag) + del CONSUMERS[consumer_tag] def consume(self, limit=None): + global CONSUMERS + num = 0 while True: - item = self.get(self.current_queue) - if item: - self.current_callback(item) - raise StopIteration() - greenthread.sleep(0) + for (queue, callback) in CONSUMERS.itervalues(): + item = self.get(queue) + if item: + callback(item) + num += 1 + yield + if limit and num == limit: + raise StopIteration() + greenthread.sleep(0.1) def get(self, queue, no_ack=False): global QUEUES @@ -134,5 +147,7 @@ class Backend(base.BaseBackend): def reset_all(): global EXCHANGES global QUEUES + global CONSUMERS EXCHANGES = {} QUEUES = {} + CONSUMERS = {} diff --git a/nova/flags.py b/nova/flags.py index 32cb6efa8..57a4ecf2f 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -110,7 +110,7 @@ class FlagValues(gflags.FlagValues): return name in self.__dict__['__dirty'] def ClearDirty(self): - self.__dict__['__is_dirty'] = [] + self.__dict__['__dirty'] = [] def WasAlreadyParsed(self): return self.__dict__['__was_already_parsed'] @@ -119,11 +119,12 @@ class FlagValues(gflags.FlagValues): if '__stored_argv' not in self.__dict__: return new_flags = FlagValues(self) - for k in self.__dict__['__dirty']: + for k in self.FlagDict().iterkeys(): new_flags[k] = gflags.FlagValues.__getitem__(self, k) + new_flags.Reset() new_flags(self.__dict__['__stored_argv']) - for k in self.__dict__['__dirty']: + for k in new_flags.FlagDict().iterkeys(): setattr(self, k, getattr(new_flags, k)) self.ClearDirty() @@ -269,8 +270,10 @@ DEFINE_list('region_list', DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake') DEFINE_string('aws_access_key_id', 'admin', 'AWS Access ID') DEFINE_string('aws_secret_access_key', 'admin', 'AWS Access Key') -DEFINE_integer('glance_port', 9292, 'glance port') -DEFINE_string('glance_host', '$my_ip', 'glance host') +# NOTE(sirp): my_ip interpolation doesn't work within nested structures +DEFINE_list('glance_api_servers', + ['%s:9292' % _get_my_ip()], + 'list of glance api servers available to nova (host:port)') DEFINE_integer('s3_port', 3333, 's3 port') DEFINE_string('s3_host', '$my_ip', 's3 host (for infrastructure)') DEFINE_string('s3_dmz', '$my_ip', 's3 dmz ip (for instances)') @@ -295,6 +298,7 @@ DEFINE_bool('fake_network', False, 'should we use fake network devices and addresses') DEFINE_string('rabbit_host', 'localhost', 'rabbit host') DEFINE_integer('rabbit_port', 5672, 'rabbit port') +DEFINE_bool('rabbit_use_ssl', False, 'connect over SSL') DEFINE_string('rabbit_userid', 'guest', 'rabbit userid') DEFINE_string('rabbit_password', 'guest', 'rabbit password') DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host') @@ -360,7 +364,7 @@ DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', 'Manager for scheduler') # The service to use for image search and retrieval -DEFINE_string('image_service', 'nova.image.local.LocalImageService', +DEFINE_string('image_service', 'nova.image.glance.GlanceImageService', 'The service to use for retrieving and searching for images.') DEFINE_string('host', socket.gethostname(), @@ -379,3 +383,5 @@ DEFINE_string('zone_name', 'nova', 'name of this zone') DEFINE_list('zone_capabilities', ['hypervisor=xenserver;kvm', 'os=linux;windows'], 'Key/Multi-value list representng capabilities of this zone') +DEFINE_string('build_plan_encryption_key', None, + '128bit (hex) encryption key for scheduler build plans.') diff --git a/nova/image/__init__.py b/nova/image/__init__.py index e69de29bb..a27d649d4 100644 --- a/nova/image/__init__.py +++ b/nova/image/__init__.py @@ -0,0 +1,98 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from urlparse import urlparse + +import nova +from nova import exception +from nova import utils +from nova import flags +from nova.image import glance as glance_image_service + +FLAGS = flags.FLAGS + + +GlanceClient = utils.import_class('glance.client.Client') + + +def _parse_image_ref(image_href): + """Parse an image href into composite parts. + + :param image_href: href of an image + :returns: a tuple of the form (image_id, host, port) + + """ + o = urlparse(image_href) + port = o.port or 80 + host = o.netloc.split(':', 1)[0] + image_id = int(o.path.split('/')[-1]) + return (image_id, host, port) + + +def get_default_image_service(): + ImageService = utils.import_class(FLAGS.image_service) + return ImageService() + + +# FIXME(sirp): perhaps this should be moved to nova/images/glance so that we +# keep Glance specific code together for the most part +def get_glance_client(image_href): + """Get the correct glance client and id for the given image_href. + + The image_href param can be an href of the form + http://myglanceserver:9292/images/42, or just an int such as 42. If the + image_href is an int, then flags are used to create the default + glance client. + + :param image_href: image ref/id for an image + :returns: a tuple of the form (glance_client, image_id) + + """ + image_href = image_href or 0 + if str(image_href).isdigit(): + glance_host, glance_port = \ + glance_image_service.pick_glance_api_server() + glance_client = GlanceClient(glance_host, glance_port) + return (glance_client, int(image_href)) + + try: + (image_id, host, port) = _parse_image_ref(image_href) + except: + raise exception.InvalidImageRef(image_href=image_href) + glance_client = GlanceClient(host, port) + return (glance_client, image_id) + + +def get_image_service(image_href): + """Get the proper image_service and id for the given image_href. + + The image_href param can be an href of the form + http://myglanceserver:9292/images/42, or just an int such as 42. If the + image_href is an int, then the default image service is returned. + + :param image_href: image ref/id for an image + :returns: a tuple of the form (image_service, image_id) + + """ + image_href = image_href or 0 + if str(image_href).isdigit(): + return (get_default_image_service(), int(image_href)) + + (glance_client, image_id) = get_glance_client(image_href) + image_service = nova.image.glance.GlanceImageService(glance_client) + return (image_service, image_id) diff --git a/nova/image/fake.py b/nova/image/fake.py index b400b2adb..c4b3d5fd6 100644 --- a/nova/image/fake.py +++ b/nova/image/fake.py @@ -19,6 +19,7 @@ import copy import datetime +import random from nova import exception from nova import flags @@ -32,7 +33,7 @@ LOG = logging.getLogger('nova.image.fake') FLAGS = flags.FLAGS -class FakeImageService(service.BaseImageService): +class _FakeImageService(service.BaseImageService): """Mock (fake) image service for unit testing.""" def __init__(self): @@ -40,7 +41,18 @@ class FakeImageService(service.BaseImageService): # NOTE(justinsb): The OpenStack API can't upload an image? # So, make sure we've got one.. timestamp = datetime.datetime(2011, 01, 01, 01, 02, 03) - image = {'id': '123456', + image1 = {'id': '123456', + 'name': 'fakeimage123456', + 'created_at': timestamp, + 'updated_at': timestamp, + 'status': 'active', + 'container_format': 'ami', + 'disk_format': 'raw', + 'properties': {'kernel_id': FLAGS.null_kernel, + 'ramdisk_id': FLAGS.null_kernel, + 'architecture': 'x86_64'}} + + image2 = {'id': 'fake', 'name': 'fakeimage123456', 'created_at': timestamp, 'updated_at': timestamp, @@ -49,14 +61,49 @@ class FakeImageService(service.BaseImageService): 'disk_format': 'raw', 'properties': {'kernel_id': FLAGS.null_kernel, 'ramdisk_id': FLAGS.null_kernel}} - self.create(None, image) - super(FakeImageService, self).__init__() - def index(self, context): + image3 = {'id': '2', + 'name': 'fakeimage123456', + 'created_at': timestamp, + 'updated_at': timestamp, + 'status': 'active', + 'container_format': 'ami', + 'disk_format': 'raw', + 'properties': {'kernel_id': FLAGS.null_kernel, + 'ramdisk_id': FLAGS.null_kernel}} + + image4 = {'id': '1', + 'name': 'fakeimage123456', + 'created_at': timestamp, + 'updated_at': timestamp, + 'status': 'active', + 'container_format': 'ami', + 'disk_format': 'raw', + 'properties': {'kernel_id': FLAGS.null_kernel, + 'ramdisk_id': FLAGS.null_kernel}} + + image5 = {'id': '3', + 'name': 'fakeimage123456', + 'created_at': timestamp, + 'updated_at': timestamp, + 'status': 'active', + 'container_format': 'ami', + 'disk_format': 'raw', + 'properties': {'kernel_id': FLAGS.null_kernel, + 'ramdisk_id': FLAGS.null_kernel}} + + self.create(None, image1) + self.create(None, image2) + self.create(None, image3) + self.create(None, image4) + self.create(None, image5) + super(_FakeImageService, self).__init__() + + def index(self, context, filters=None, marker=None, limit=None): """Returns list of images.""" return copy.deepcopy(self.images.values()) - def detail(self, context): + def detail(self, context, filters=None, marker=None, limit=None): """Return list of detailed image information.""" return copy.deepcopy(self.images.values()) @@ -66,36 +113,49 @@ class FakeImageService(service.BaseImageService): Returns a dict containing image data for the given opaque image id. """ - image_id = int(image_id) - image = self.images.get(image_id) + image = self.images.get(str(image_id)) if image: return copy.deepcopy(image) LOG.warn('Unable to find image id %s. Have images: %s', image_id, self.images) raise exception.ImageNotFound(image_id=image_id) - def create(self, context, data): + def show_by_name(self, context, name): + """Returns a dict containing image data for the given name.""" + images = copy.deepcopy(self.images.values()) + for image in images: + if name == image.get('name'): + return image + raise exception.ImageNotFound(image_id=name) + + def create(self, context, metadata, data=None): """Store the image data and return the new image id. :raises: Duplicate if the image already exist. """ - image_id = int(data['id']) + try: + image_id = metadata['id'] + except KeyError: + image_id = random.randint(0, 2 ** 31 - 1) + image_id = str(image_id) + if self.images.get(image_id): raise exception.Duplicate() - self.images[image_id] = copy.deepcopy(data) + metadata['id'] = image_id + self.images[image_id] = copy.deepcopy(metadata) + return self.images[image_id] - def update(self, context, image_id, data): + def update(self, context, image_id, metadata, data=None): """Replace the contents of the given image with the new data. :raises: ImageNotFound if the image does not exist. """ - image_id = int(image_id) if not self.images.get(image_id): raise exception.ImageNotFound(image_id=image_id) - self.images[image_id] = copy.deepcopy(data) + self.images[image_id] = copy.deepcopy(metadata) def delete(self, context, image_id): """Delete the given image. @@ -103,7 +163,6 @@ class FakeImageService(service.BaseImageService): :raises: ImageNotFound if the image does not exist. """ - image_id = int(image_id) removed = self.images.pop(image_id, None) if not removed: raise exception.ImageNotFound(image_id=image_id) @@ -111,3 +170,9 @@ class FakeImageService(service.BaseImageService): def delete_all(self): """Clears out all images.""" self.images.clear() + +_fakeImageService = _FakeImageService() + + +def FakeImageService(): + return _fakeImageService diff --git a/nova/image/glance.py b/nova/image/glance.py index 193e37273..55d948a32 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -20,6 +20,7 @@ from __future__ import absolute_import import datetime +import random from glance.common import exception as glance_exception @@ -39,11 +40,26 @@ FLAGS = flags.FLAGS GlanceClient = utils.import_class('glance.client.Client') +def pick_glance_api_server(): + """Return which Glance API server to use for the request + + This method provides a very primitive form of load-balancing suitable for + testing and sandbox environments. In production, it would be better to use + one IP and route that to a real load-balancer. + + Returns (host, port) + """ + host_port = random.choice(FLAGS.glance_api_servers) + host, port_str = host_port.split(':') + port = int(port_str) + return host, port + + class GlanceImageService(service.BaseImageService): """Provides storage and retrieval of disk image objects within Glance.""" GLANCE_ONLY_ATTRS = ['size', 'location', 'disk_format', - 'container_format'] + 'container_format', 'checksum'] # NOTE(sirp): Overriding to use _translate_to_service provided by # BaseImageService @@ -51,30 +67,43 @@ class GlanceImageService(service.BaseImageService): GLANCE_ONLY_ATTRS def __init__(self, client=None): - # FIXME(sirp): can we avoid dependency-injection here by using - # stubbing out a fake? - if client is None: - self.client = GlanceClient(FLAGS.glance_host, FLAGS.glance_port) - else: - self.client = client - - def index(self, context): + self._client = client + + def _get_client(self): + # NOTE(sirp): we want to load balance each request across glance + # servers. Since GlanceImageService is a long-lived object, `client` + # is made to choose a new server each time via this property. + if self._client is not None: + return self._client + glance_host, glance_port = pick_glance_api_server() + return GlanceClient(glance_host, glance_port) + + def _set_client(self, client): + self._client = client + + client = property(_get_client, _set_client) + + def index(self, context, filters=None, marker=None, limit=None): """Calls out to Glance for a list of images available.""" # NOTE(sirp): We need to use `get_images_detailed` and not # `get_images` here because we need `is_public` and `properties` # included so we can filter by user filtered = [] - image_metas = self.client.get_images_detailed() + image_metas = self.client.get_images_detailed(filters=filters, + marker=marker, + limit=limit) for image_meta in image_metas: if self._is_image_available(context, image_meta): meta_subset = utils.subset_dict(image_meta, ('id', 'name')) filtered.append(meta_subset) return filtered - def detail(self, context): + def detail(self, context, filters=None, marker=None, limit=None): """Calls out to Glance for a list of detailed image information.""" filtered = [] - image_metas = self.client.get_images_detailed() + image_metas = self.client.get_images_detailed(filters=filters, + marker=marker, + limit=limit) for image_meta in image_metas: if self._is_image_available(context, image_meta): base_image_meta = self._translate_to_base(image_meta) diff --git a/nova/image/local.py b/nova/image/local.py deleted file mode 100644 index 918180bae..000000000 --- a/nova/image/local.py +++ /dev/null @@ -1,165 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import os.path -import random -import shutil - -from nova import exception -from nova import flags -from nova import log as logging -from nova import utils -from nova.image import service - - -FLAGS = flags.FLAGS -flags.DEFINE_string('images_path', '$state_path/images', - 'path to decrypted images') - - -LOG = logging.getLogger('nova.image.local') - - -class LocalImageService(service.BaseImageService): - """Image service storing images to local disk. - - It assumes that image_ids are integers. - - """ - - def __init__(self): - self._path = FLAGS.images_path - - def _path_to(self, image_id, fname='info.json'): - if fname: - return os.path.join(self._path, '%08x' % int(image_id), fname) - return os.path.join(self._path, '%08x' % int(image_id)) - - def _ids(self): - """The list of all image ids.""" - images = [] - for image_dir in os.listdir(self._path): - try: - unhexed_image_id = int(image_dir, 16) - except ValueError: - LOG.error(_('%s is not in correct directory naming format') - % image_dir) - else: - images.append(unhexed_image_id) - return images - - def index(self, context): - filtered = [] - image_metas = self.detail(context) - for image_meta in image_metas: - meta = utils.subset_dict(image_meta, ('id', 'name')) - filtered.append(meta) - return filtered - - def detail(self, context): - images = [] - for image_id in self._ids(): - try: - image = self.show(context, image_id) - images.append(image) - except exception.NotFound: - continue - return images - - def show(self, context, image_id): - try: - with open(self._path_to(image_id)) as metadata_file: - image_meta = json.load(metadata_file) - if not self._is_image_available(context, image_meta): - raise exception.ImageNotFound(image_id=image_id) - return image_meta - except (IOError, ValueError): - raise exception.ImageNotFound(image_id=image_id) - - def show_by_name(self, context, name): - """Returns a dict containing image data for the given name.""" - # NOTE(vish): Not very efficient, but the local image service - # is for testing so it should be fine. - images = self.detail(context) - image = None - for cantidate in images: - if name == cantidate.get('name'): - image = cantidate - break - if image is None: - raise exception.ImageNotFound(image_id=name) - return image - - def get(self, context, image_id, data): - """Get image and metadata.""" - try: - with open(self._path_to(image_id)) as metadata_file: - metadata = json.load(metadata_file) - with open(self._path_to(image_id, 'image')) as image_file: - shutil.copyfileobj(image_file, data) - except (IOError, ValueError): - raise exception.ImageNotFound(image_id=image_id) - return metadata - - def create(self, context, metadata, data=None): - """Store the image data and return the new image.""" - image_id = random.randint(0, 2 ** 31 - 1) - image_path = self._path_to(image_id, None) - if not os.path.exists(image_path): - os.mkdir(image_path) - return self._store(context, image_id, metadata, data) - - def update(self, context, image_id, metadata, data=None): - """Replace the contents of the given image with the new data.""" - # NOTE(vish): show is to check if image is available - self.show(context, image_id) - return self._store(context, image_id, metadata, data) - - def _store(self, context, image_id, metadata, data=None): - metadata['id'] = image_id - try: - if data: - location = self._path_to(image_id, 'image') - with open(location, 'w') as image_file: - shutil.copyfileobj(data, image_file) - # NOTE(vish): update metadata similarly to glance - metadata['status'] = 'active' - metadata['location'] = location - with open(self._path_to(image_id), 'w') as metadata_file: - json.dump(metadata, metadata_file) - except (IOError, ValueError): - raise exception.ImageNotFound(image_id=image_id) - return metadata - - def delete(self, context, image_id): - """Delete the given image. - - :raises: ImageNotFound if the image does not exist. - - """ - # NOTE(vish): show is to check if image is available - self.show(context, image_id) - try: - shutil.rmtree(self._path_to(image_id, None)) - except (IOError, ValueError): - raise exception.ImageNotFound(image_id=image_id) - - def delete_all(self): - """Clears out all images in local directory.""" - for image_id in self._ids(): - shutil.rmtree(self._path_to(image_id, None)) diff --git a/nova/image/s3.py b/nova/image/s3.py index c38c58d95..9e95bd698 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -31,12 +31,15 @@ import eventlet from nova import crypto from nova import exception from nova import flags +from nova import image +from nova import log as logging from nova import utils from nova.auth import manager from nova.image import service from nova.api.ec2 import ec2utils +LOG = logging.getLogger("nova.image.s3") FLAGS = flags.FLAGS flags.DEFINE_string('image_decryption_dir', '/tmp', 'parent dir for tempdir used for image decryption') @@ -46,9 +49,7 @@ class S3ImageService(service.BaseImageService): """Wraps an existing image service to support s3 based register.""" def __init__(self, service=None, *args, **kwargs): - if service is None: - service = utils.import_object(FLAGS.image_service) - self.service = service + self.service = service or image.get_default_image_service() self.service.__init__(*args, **kwargs) def create(self, context, metadata, data=None): @@ -161,43 +162,83 @@ class S3ImageService(service.BaseImageService): def delayed_create(): """This handles the fetching and decrypting of the part files.""" - parts = [] - for fn_element in manifest.find('image').getiterator('filename'): - part = self._download_file(bucket, fn_element.text, image_path) - parts.append(part) - - # NOTE(vish): this may be suboptimal, should we use cat? - encrypted_filename = os.path.join(image_path, 'image.encrypted') - with open(encrypted_filename, 'w') as combined: - for filename in parts: - with open(filename) as part: - shutil.copyfileobj(part, combined) - - metadata['properties']['image_state'] = 'decrypting' + metadata['properties']['image_state'] = 'downloading' self.service.update(context, image_id, metadata) - hex_key = manifest.find('image/ec2_encrypted_key').text - encrypted_key = binascii.a2b_hex(hex_key) - hex_iv = manifest.find('image/ec2_encrypted_iv').text - encrypted_iv = binascii.a2b_hex(hex_iv) + try: + parts = [] + elements = manifest.find('image').getiterator('filename') + for fn_element in elements: + part = self._download_file(bucket, + fn_element.text, + image_path) + parts.append(part) + + # NOTE(vish): this may be suboptimal, should we use cat? + enc_filename = os.path.join(image_path, 'image.encrypted') + with open(enc_filename, 'w') as combined: + for filename in parts: + with open(filename) as part: + shutil.copyfileobj(part, combined) + + except Exception: + LOG.error(_("Failed to download %(image_location)s " + "to %(image_path)s"), locals()) + metadata['properties']['image_state'] = 'failed_download' + self.service.update(context, image_id, metadata) + raise - # FIXME(vish): grab key from common service so this can run on - # any host. - cloud_pk = crypto.key_path(context.project_id) + metadata['properties']['image_state'] = 'decrypting' + self.service.update(context, image_id, metadata) - decrypted_filename = os.path.join(image_path, 'image.tar.gz') - self._decrypt_image(encrypted_filename, encrypted_key, - encrypted_iv, cloud_pk, decrypted_filename) + try: + hex_key = manifest.find('image/ec2_encrypted_key').text + encrypted_key = binascii.a2b_hex(hex_key) + hex_iv = manifest.find('image/ec2_encrypted_iv').text + encrypted_iv = binascii.a2b_hex(hex_iv) + + # FIXME(vish): grab key from common service so this can run on + # any host. + cloud_pk = crypto.key_path(context.project_id) + + dec_filename = os.path.join(image_path, 'image.tar.gz') + self._decrypt_image(enc_filename, encrypted_key, + encrypted_iv, cloud_pk, + dec_filename) + except Exception: + LOG.error(_("Failed to decrypt %(image_location)s " + "to %(image_path)s"), locals()) + metadata['properties']['image_state'] = 'failed_decrypt' + self.service.update(context, image_id, metadata) + raise metadata['properties']['image_state'] = 'untarring' self.service.update(context, image_id, metadata) - unz_filename = self._untarzip_image(image_path, decrypted_filename) + try: + unz_filename = self._untarzip_image(image_path, dec_filename) + except Exception: + LOG.error(_("Failed to untar %(image_location)s " + "to %(image_path)s"), locals()) + metadata['properties']['image_state'] = 'failed_untar' + self.service.update(context, image_id, metadata) + raise metadata['properties']['image_state'] = 'uploading' - with open(unz_filename) as image_file: - self.service.update(context, image_id, metadata, image_file) + self.service.update(context, image_id, metadata) + try: + with open(unz_filename) as image_file: + self.service.update(context, image_id, + metadata, image_file) + except Exception: + LOG.error(_("Failed to upload %(image_location)s " + "to %(image_path)s"), locals()) + metadata['properties']['image_state'] = 'failed_upload' + self.service.update(context, image_id, metadata) + raise + metadata['properties']['image_state'] = 'available' + metadata['status'] = 'active' self.service.update(context, image_id, metadata) shutil.rmtree(image_path) diff --git a/nova/image/service.py b/nova/image/service.py index ab6749049..5361cfc89 100644 --- a/nova/image/service.py +++ b/nova/image/service.py @@ -46,7 +46,7 @@ class BaseImageService(object): # the ImageService subclass SERVICE_IMAGE_ATTRS = [] - def index(self, context): + def index(self, context, *args, **kwargs): """List images. :returns: a sequence of mappings with the following signature @@ -55,7 +55,7 @@ class BaseImageService(object): """ raise NotImplementedError - def detail(self, context): + def detail(self, context, *args, **kwargs): """Detailed information about an images. :returns: a sequence of mappings with the following signature diff --git a/nova/log.py b/nova/log.py index 096279f7c..6909916a1 100644 --- a/nova/log.py +++ b/nova/log.py @@ -35,6 +35,7 @@ import os import sys import traceback +import nova from nova import flags from nova import version @@ -63,6 +64,7 @@ flags.DEFINE_list('default_log_levels', 'eventlet.wsgi.server=WARN'], 'list of logger=LEVEL pairs') flags.DEFINE_bool('use_syslog', False, 'output to syslog') +flags.DEFINE_bool('publish_errors', False, 'publish error events') flags.DEFINE_string('logfile', None, 'output to named file') @@ -258,12 +260,20 @@ class NovaRootLogger(NovaLogger): else: self.removeHandler(self.filelog) self.addHandler(self.streamlog) + if FLAGS.publish_errors: + self.addHandler(PublishErrorsHandler(ERROR)) if FLAGS.verbose: self.setLevel(DEBUG) else: self.setLevel(INFO) +class PublishErrorsHandler(logging.Handler): + def emit(self, record): + nova.notifier.api.notify('nova.error.publisher', 'error_notification', + nova.notifier.api.ERROR, dict(error=record.msg)) + + def handle_exception(type, value, tb): extra = {} if FLAGS.verbose: diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 815cd29c3..6c5a6f1ce 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -20,6 +20,7 @@ import calendar import inspect +import netaddr import os from nova import db @@ -27,7 +28,6 @@ from nova import exception from nova import flags from nova import log as logging from nova import utils -from IPy import IP LOG = logging.getLogger("nova.linux_net") @@ -191,6 +191,13 @@ class IptablesTable(object): {'chain': chain, 'rule': rule, 'top': top, 'wrap': wrap}) + def empty_chain(self, chain, wrap=True): + """Remove all rules from a chain.""" + chained_rules = [rule for rule in self.rules + if rule.chain == chain and rule.wrap == wrap] + for rule in chained_rules: + self.rules.remove(rule) + class IptablesManager(object): """Wrapper for iptables. @@ -700,7 +707,7 @@ def _dnsmasq_cmd(net): '--listen-address=%s' % net['gateway'], '--except-interface=lo', '--dhcp-range=%s,static,120s' % net['dhcp_start'], - '--dhcp-lease-max=%s' % IP(net['cidr']).len(), + '--dhcp-lease-max=%s' % len(netaddr.IPNetwork(net['cidr'])), '--dhcp-hostsfile=%s' % _dhcp_file(net['bridge'], 'conf'), '--dhcp-script=%s' % FLAGS.dhcpbridge, '--leasefile-ro'] diff --git a/nova/network/manager.py b/nova/network/manager.py index 5a6fdde5a..bf0456522 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -45,10 +45,9 @@ topologies. All of the network commands are issued to a subclass of import datetime import math +import netaddr import socket -import IPy - from nova import context from nova import db from nova import exception @@ -86,6 +85,7 @@ flags.DEFINE_string('floating_range', '4.4.4.0/24', 'Floating IP address block') flags.DEFINE_string('fixed_range', '10.0.0.0/8', 'Fixed IP address block') flags.DEFINE_string('fixed_range_v6', 'fd00::/48', 'Fixed IPv6 address block') +flags.DEFINE_string('gateway_v6', None, 'Default IPv6 gateway') flags.DEFINE_integer('cnt_vpn_clients', 0, 'Number of addresses reserved for vpn clients') flags.DEFINE_string('network_driver', 'nova.network.linux_net', @@ -235,7 +235,7 @@ class NetworkManager(manager.SchedulerDependentManager): inst_addr = instance_ref['mac_address'] raise exception.Error(_('IP %(address)s leased to bad mac' ' %(inst_addr)s vs %(mac)s') % locals()) - now = datetime.datetime.utcnow() + now = utils.utcnow() self.db.fixed_ip_update(context, fixed_ip_ref['address'], {'leased': True, @@ -292,10 +292,10 @@ class NetworkManager(manager.SchedulerDependentManager): return host def create_networks(self, context, cidr, num_networks, network_size, - cidr_v6, label, *args, **kwargs): + cidr_v6, gateway_v6, label, *args, **kwargs): """Create networks based on parameters.""" - fixed_net = IPy.IP(cidr) - fixed_net_v6 = IPy.IP(cidr_v6) + fixed_net = netaddr.IPNetwork(cidr) + fixed_net_v6 = netaddr.IPNetwork(cidr_v6) significant_bits_v6 = 64 network_size_v6 = 1 << 64 count = 1 @@ -304,15 +304,15 @@ class NetworkManager(manager.SchedulerDependentManager): start_v6 = index * network_size_v6 significant_bits = 32 - int(math.log(network_size, 2)) cidr = '%s/%s' % (fixed_net[start], significant_bits) - project_net = IPy.IP(cidr) + project_net = netaddr.IPNetwork(cidr) net = {} net['bridge'] = FLAGS.flat_network_bridge net['dns'] = FLAGS.flat_network_dns net['cidr'] = cidr - net['netmask'] = str(project_net.netmask()) - net['gateway'] = str(project_net[1]) - net['broadcast'] = str(project_net.broadcast()) - net['dhcp_start'] = str(project_net[2]) + net['netmask'] = str(project_net.netmask) + net['gateway'] = str(list(project_net)[1]) + net['broadcast'] = str(project_net.broadcast) + net['dhcp_start'] = str(list(project_net)[2]) if num_networks > 1: net['label'] = '%s_%d' % (label, count) else: @@ -323,9 +323,16 @@ class NetworkManager(manager.SchedulerDependentManager): cidr_v6 = '%s/%s' % (fixed_net_v6[start_v6], significant_bits_v6) net['cidr_v6'] = cidr_v6 - project_net_v6 = IPy.IP(cidr_v6) - net['gateway_v6'] = str(project_net_v6[1]) - net['netmask_v6'] = str(project_net_v6.prefixlen()) + + project_net_v6 = netaddr.IPNetwork(cidr_v6) + + if gateway_v6: + # use a pre-defined gateway if one is provided + net['gateway_v6'] = str(list(gateway_v6)[1]) + else: + net['gateway_v6'] = str(list(project_net_v6)[1]) + + net['netmask_v6'] = str(project_net_v6._prefixlen) network_ref = self.db.network_create_safe(context, net) @@ -349,7 +356,7 @@ class NetworkManager(manager.SchedulerDependentManager): # to properties of the manager class? bottom_reserved = self._bottom_reserved_ips top_reserved = self._top_reserved_ips - project_net = IPy.IP(network_ref['cidr']) + project_net = netaddr.IPNetwork(network_ref['cidr']) num_ips = len(project_net) for index in range(num_ips): address = str(project_net[index]) @@ -539,13 +546,13 @@ class VlanManager(NetworkManager): ' the vlan start cannot be greater' ' than 4094')) - fixed_net = IPy.IP(cidr) - if fixed_net.len() < num_networks * network_size: + fixed_net = netaddr.IPNetwork(cidr) + if len(fixed_net) < num_networks * network_size: raise ValueError(_('The network range is not big enough to fit ' '%(num_networks)s. Network size is %(network_size)s' % locals())) - fixed_net_v6 = IPy.IP(cidr_v6) + fixed_net_v6 = netaddr.IPNetwork(cidr_v6) network_size_v6 = 1 << 64 significant_bits_v6 = 64 for index in range(num_networks): @@ -554,14 +561,14 @@ class VlanManager(NetworkManager): start_v6 = index * network_size_v6 significant_bits = 32 - int(math.log(network_size, 2)) cidr = "%s/%s" % (fixed_net[start], significant_bits) - project_net = IPy.IP(cidr) + project_net = netaddr.IPNetwork(cidr) net = {} net['cidr'] = cidr - net['netmask'] = str(project_net.netmask()) - net['gateway'] = str(project_net[1]) - net['broadcast'] = str(project_net.broadcast()) - net['vpn_private_address'] = str(project_net[2]) - net['dhcp_start'] = str(project_net[3]) + net['netmask'] = str(project_net.netmask) + net['gateway'] = str(list(project_net)[1]) + net['broadcast'] = str(project_net.broadcast) + net['vpn_private_address'] = str(list(project_net)[2]) + net['dhcp_start'] = str(list(project_net)[3]) net['vlan'] = vlan net['bridge'] = 'br%s' % vlan if(FLAGS.use_ipv6): diff --git a/nova/network/vmwareapi_net.py b/nova/network/vmwareapi_net.py index 373060add..04210c011 100644 --- a/nova/network/vmwareapi_net.py +++ b/nova/network/vmwareapi_net.py @@ -30,9 +30,7 @@ LOG = logging.getLogger("nova.network.vmwareapi_net") FLAGS = flags.FLAGS -flags.DEFINE_string('vlan_interface', 'vmnic0', - 'Physical network adapter name in VMware ESX host for ' - 'vlan networking') +FLAGS['vlan_interface'].SetDefault('vmnic0') def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): diff --git a/nova/network/xenapi_net.py b/nova/network/xenapi_net.py index 709ef7f34..af295a4f8 100644 --- a/nova/network/xenapi_net.py +++ b/nova/network/xenapi_net.py @@ -56,8 +56,10 @@ def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): 'other_config': {}} network_ref = session.call_xenapi('network.create', network_rec) # 2 - find PIF for VLAN - expr = "field 'device' = '%s' and \ - field 'VLAN' = '-1'" % FLAGS.vlan_interface + # NOTE(salvatore-orlando): using double quotes inside single quotes + # as xapi filter only support tokens in double quotes + expr = 'field "device" = "%s" and \ + field "VLAN" = "-1"' % FLAGS.vlan_interface pifs = session.call_xenapi('PIF.get_all_records_where', expr) pif_ref = None # Multiple PIF are ok: we are dealing with a pool diff --git a/nova/notifier/api.py b/nova/notifier/api.py index a3e7a039e..d49517c8b 100644 --- a/nova/notifier/api.py +++ b/nova/notifier/api.py @@ -11,9 +11,8 @@ # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations -# under the License.import datetime +# under the License. -import datetime import uuid from nova import flags @@ -64,7 +63,7 @@ def notify(publisher_id, event_type, priority, payload): {'message_id': str(uuid.uuid4()), 'publisher_id': 'compute.host1', - 'timestamp': datetime.datetime.utcnow(), + 'timestamp': utils.utcnow(), 'priority': 'WARN', 'event_type': 'compute.create_instance', 'payload': {'instance_id': 12, ... }} @@ -79,5 +78,5 @@ def notify(publisher_id, event_type, priority, payload): event_type=event_type, priority=priority, payload=payload, - timestamp=str(datetime.datetime.utcnow())) + timestamp=str(utils.utcnow())) driver.notify(msg) diff --git a/nova/objectstore/s3server.py b/nova/objectstore/s3server.py index dd6327c8f..76025a1e3 100644 --- a/nova/objectstore/s3server.py +++ b/nova/objectstore/s3server.py @@ -81,7 +81,7 @@ class S3Application(wsgi.Router): super(S3Application, self).__init__(mapper) -class BaseRequestHandler(wsgi.Controller): +class BaseRequestHandler(object): """Base class emulating Tornado's web framework pattern in WSGI. This is a direct port of Tornado's implementation, so some key decisions diff --git a/nova/quota.py b/nova/quota.py index a93cd0766..58766e846 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -28,6 +28,8 @@ flags.DEFINE_integer('quota_instances', 10, 'number of instances allowed per project') flags.DEFINE_integer('quota_cores', 20, 'number of instance cores allowed per project') +flags.DEFINE_integer('quota_ram', 50 * 1024, + 'megabytes of instance ram allowed per project') flags.DEFINE_integer('quota_volumes', 10, 'number of volumes allowed per project') flags.DEFINE_integer('quota_gigabytes', 1000, @@ -44,14 +46,28 @@ flags.DEFINE_integer('quota_max_injected_file_path_bytes', 255, 'number of bytes allowed per injected file path') -def get_quota(context, project_id): - rval = {'instances': FLAGS.quota_instances, - 'cores': FLAGS.quota_cores, - 'volumes': FLAGS.quota_volumes, - 'gigabytes': FLAGS.quota_gigabytes, - 'floating_ips': FLAGS.quota_floating_ips, - 'metadata_items': FLAGS.quota_metadata_items} - +def _get_default_quotas(): + defaults = { + 'instances': FLAGS.quota_instances, + 'cores': FLAGS.quota_cores, + 'ram': FLAGS.quota_ram, + 'volumes': FLAGS.quota_volumes, + 'gigabytes': FLAGS.quota_gigabytes, + 'floating_ips': FLAGS.quota_floating_ips, + 'metadata_items': FLAGS.quota_metadata_items, + 'injected_files': FLAGS.quota_max_injected_files, + 'injected_file_content_bytes': + FLAGS.quota_max_injected_file_content_bytes, + } + # -1 in the quota flags means unlimited + for key in defaults.keys(): + if defaults[key] == -1: + defaults[key] = None + return defaults + + +def get_project_quotas(context, project_id): + rval = _get_default_quotas() quota = db.quota_get_all_by_project(context, project_id) for key in rval.keys(): if key in quota: @@ -65,71 +81,81 @@ def _get_request_allotment(requested, used, quota): return quota - used -def allowed_instances(context, num_instances, instance_type): - """Check quota and return min(num_instances, allowed_instances).""" +def allowed_instances(context, requested_instances, instance_type): + """Check quota and return min(requested_instances, allowed_instances).""" project_id = context.project_id context = context.elevated() - num_cores = num_instances * instance_type['vcpus'] - used_instances, used_cores = db.instance_data_get_for_project(context, - project_id) - quota = get_quota(context, project_id) - allowed_instances = _get_request_allotment(num_instances, used_instances, + requested_cores = requested_instances * instance_type['vcpus'] + requested_ram = requested_instances * instance_type['memory_mb'] + usage = db.instance_data_get_for_project(context, project_id) + used_instances, used_cores, used_ram = usage + quota = get_project_quotas(context, project_id) + allowed_instances = _get_request_allotment(requested_instances, + used_instances, quota['instances']) - allowed_cores = _get_request_allotment(num_cores, used_cores, + allowed_cores = _get_request_allotment(requested_cores, used_cores, quota['cores']) + allowed_ram = _get_request_allotment(requested_ram, used_ram, quota['ram']) allowed_instances = min(allowed_instances, - int(allowed_cores // instance_type['vcpus'])) - return min(num_instances, allowed_instances) + allowed_cores // instance_type['vcpus'], + allowed_ram // instance_type['memory_mb']) + return min(requested_instances, allowed_instances) -def allowed_volumes(context, num_volumes, size): - """Check quota and return min(num_volumes, allowed_volumes).""" +def allowed_volumes(context, requested_volumes, size): + """Check quota and return min(requested_volumes, allowed_volumes).""" project_id = context.project_id context = context.elevated() size = int(size) - num_gigabytes = num_volumes * size + requested_gigabytes = requested_volumes * size used_volumes, used_gigabytes = db.volume_data_get_for_project(context, project_id) - quota = get_quota(context, project_id) - allowed_volumes = _get_request_allotment(num_volumes, used_volumes, + quota = get_project_quotas(context, project_id) + allowed_volumes = _get_request_allotment(requested_volumes, used_volumes, quota['volumes']) - allowed_gigabytes = _get_request_allotment(num_gigabytes, used_gigabytes, + allowed_gigabytes = _get_request_allotment(requested_gigabytes, + used_gigabytes, quota['gigabytes']) allowed_volumes = min(allowed_volumes, int(allowed_gigabytes // size)) - return min(num_volumes, allowed_volumes) + return min(requested_volumes, allowed_volumes) -def allowed_floating_ips(context, num_floating_ips): - """Check quota and return min(num_floating_ips, allowed_floating_ips).""" +def allowed_floating_ips(context, requested_floating_ips): + """Check quota and return min(requested, allowed) floating ips.""" project_id = context.project_id context = context.elevated() used_floating_ips = db.floating_ip_count_by_project(context, project_id) - quota = get_quota(context, project_id) - allowed_floating_ips = _get_request_allotment(num_floating_ips, + quota = get_project_quotas(context, project_id) + allowed_floating_ips = _get_request_allotment(requested_floating_ips, used_floating_ips, quota['floating_ips']) - return min(num_floating_ips, allowed_floating_ips) + return min(requested_floating_ips, allowed_floating_ips) -def allowed_metadata_items(context, num_metadata_items): - """Check quota; return min(num_metadata_items,allowed_metadata_items).""" - project_id = context.project_id - context = context.elevated() - quota = get_quota(context, project_id) - allowed_metadata_items = _get_request_allotment(num_metadata_items, 0, - quota['metadata_items']) - return min(num_metadata_items, allowed_metadata_items) +def _calculate_simple_quota(context, resource, requested): + """Check quota for resource; return min(requested, allowed).""" + quota = get_project_quotas(context, context.project_id) + allowed = _get_request_allotment(requested, 0, quota[resource]) + return min(requested, allowed) + + +def allowed_metadata_items(context, requested_metadata_items): + """Return the number of metadata items allowed.""" + return _calculate_simple_quota(context, 'metadata_items', + requested_metadata_items) -def allowed_injected_files(context): +def allowed_injected_files(context, requested_injected_files): """Return the number of injected files allowed.""" - return FLAGS.quota_max_injected_files + return _calculate_simple_quota(context, 'injected_files', + requested_injected_files) -def allowed_injected_file_content_bytes(context): +def allowed_injected_file_content_bytes(context, requested_bytes): """Return the number of bytes allowed per injected file content.""" - return FLAGS.quota_max_injected_file_content_bytes + resource = 'injected_file_content_bytes' + return _calculate_simple_quota(context, resource, requested_bytes) def allowed_injected_file_path_bytes(context): diff --git a/nova/rpc.py b/nova/rpc.py index 2116f22c3..2e78a31e7 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -28,12 +28,15 @@ import json import sys import time import traceback +import types import uuid from carrot import connection as carrot_connection from carrot import messaging from eventlet import greenpool -from eventlet import greenthread +from eventlet import pools +from eventlet import queue +import greenlet from nova import context from nova import exception @@ -47,7 +50,10 @@ LOG = logging.getLogger('nova.rpc') FLAGS = flags.FLAGS -flags.DEFINE_integer('rpc_thread_pool_size', 1024, 'Size of RPC thread pool') +flags.DEFINE_integer('rpc_thread_pool_size', 1024, + 'Size of RPC thread pool') +flags.DEFINE_integer('rpc_conn_pool_size', 30, + 'Size of RPC connection pool') class Connection(carrot_connection.BrokerConnection): @@ -59,6 +65,7 @@ class Connection(carrot_connection.BrokerConnection): if new or not hasattr(cls, '_instance'): params = dict(hostname=FLAGS.rabbit_host, port=FLAGS.rabbit_port, + ssl=FLAGS.rabbit_use_ssl, userid=FLAGS.rabbit_userid, password=FLAGS.rabbit_password, virtual_host=FLAGS.rabbit_virtual_host) @@ -90,6 +97,22 @@ class Connection(carrot_connection.BrokerConnection): return cls.instance() +class Pool(pools.Pool): + """Class that implements a Pool of Connections.""" + + # TODO(comstud): Timeout connections not used in a while + def create(self): + LOG.debug('Creating new connection') + return Connection.instance(new=True) + +# Create a ConnectionPool to use for RPC calls. We'll order the +# pool as a stack (LIFO), so that we can potentially loop through and +# timeout old unused connections at some point +ConnectionPool = Pool( + max_size=FLAGS.rpc_conn_pool_size, + order_as_stack=True) + + class Consumer(messaging.Consumer): """Consumer base class. @@ -131,7 +154,9 @@ class Consumer(messaging.Consumer): self.connection = Connection.recreate() self.backend = self.connection.create_backend() self.declare() - super(Consumer, self).fetch(no_ack, auto_ack, enable_callbacks) + return super(Consumer, self).fetch(no_ack, + auto_ack, + enable_callbacks) if self.failed_connection: LOG.error(_('Reconnected to queue')) self.failed_connection = False @@ -159,13 +184,13 @@ class AdapterConsumer(Consumer): self.pool = greenpool.GreenPool(FLAGS.rpc_thread_pool_size) super(AdapterConsumer, self).__init__(connection=connection, topic=topic) + self.register_callback(self.process_data) - def receive(self, *args, **kwargs): - self.pool.spawn_n(self._receive, *args, **kwargs) + def process_data(self, message_data, message): + """Consumer callback to call a method on a proxy object. - @exception.wrap_exception - def _receive(self, message_data, message): - """Magically looks for a method on the proxy object and calls it. + Parses the message for validity and fires off a thread to call the + proxy object method. Message data should be a dictionary with two keys: method: string representing the method to call @@ -175,8 +200,8 @@ class AdapterConsumer(Consumer): """ LOG.debug(_('received %s') % message_data) - msg_id = message_data.pop('_msg_id', None) - + # This will be popped off in _unpack_context + msg_id = message_data.get('_msg_id', None) ctxt = _unpack_context(message_data) method = message_data.get('method') @@ -188,8 +213,17 @@ class AdapterConsumer(Consumer): # we just log the message and send an error string # back to the caller LOG.warn(_('no method for message: %s') % message_data) - msg_reply(msg_id, _('No method for message: %s') % message_data) + if msg_id: + msg_reply(msg_id, + _('No method for message: %s') % message_data) return + self.pool.spawn_n(self._process_data, msg_id, ctxt, method, args) + + @exception.wrap_exception + def _process_data(self, msg_id, ctxt, method, args): + """Thread that maigcally looks for a method on the proxy + object and calls it. + """ node_func = getattr(self.proxy, str(method)) node_args = dict((str(k), v) for k, v in args.iteritems()) @@ -197,7 +231,18 @@ class AdapterConsumer(Consumer): try: rval = node_func(context=ctxt, **node_args) if msg_id: - msg_reply(msg_id, rval, None) + # Check if the result was a generator + if isinstance(rval, types.GeneratorType): + for x in rval: + msg_reply(msg_id, x, None) + else: + msg_reply(msg_id, rval, None) + + # This final None tells multicall that it is done. + msg_reply(msg_id, None, None) + elif isinstance(rval, types.GeneratorType): + # NOTE(vish): this iterates through the generator + list(rval) except Exception as e: logging.exception('Exception during message handling') if msg_id: @@ -205,11 +250,6 @@ class AdapterConsumer(Consumer): return -class Publisher(messaging.Publisher): - """Publisher base class.""" - pass - - class TopicAdapterConsumer(AdapterConsumer): """Consumes messages on a specific topic.""" @@ -242,6 +282,58 @@ class FanoutAdapterConsumer(AdapterConsumer): topic=topic, proxy=proxy) +class ConsumerSet(object): + """Groups consumers to listen on together on a single connection.""" + + def __init__(self, connection, consumer_list): + self.consumer_list = set(consumer_list) + self.consumer_set = None + self.enabled = True + self.init(connection) + + def init(self, conn): + if not conn: + conn = Connection.instance(new=True) + if self.consumer_set: + self.consumer_set.close() + self.consumer_set = messaging.ConsumerSet(conn) + for consumer in self.consumer_list: + consumer.connection = conn + # consumer.backend is set for us + self.consumer_set.add_consumer(consumer) + + def reconnect(self): + self.init(None) + + def wait(self, limit=None): + running = True + while running: + it = self.consumer_set.iterconsume(limit=limit) + if not it: + break + while True: + try: + it.next() + except StopIteration: + return + except greenlet.GreenletExit: + running = False + break + except Exception as e: + LOG.exception(_("Exception while processing consumer")) + self.reconnect() + # Break to outer loop + break + + def close(self): + self.consumer_set.close() + + +class Publisher(messaging.Publisher): + """Publisher base class.""" + pass + + class TopicPublisher(Publisher): """Publishes messages on a specific topic.""" @@ -306,16 +398,18 @@ def msg_reply(msg_id, reply=None, failure=None): LOG.error(_("Returning exception %s to caller"), message) LOG.error(tb) failure = (failure[0].__name__, str(failure[1]), tb) - conn = Connection.instance() - publisher = DirectPublisher(connection=conn, msg_id=msg_id) - try: - publisher.send({'result': reply, 'failure': failure}) - except TypeError: - publisher.send( - {'result': dict((k, repr(v)) - for k, v in reply.__dict__.iteritems()), - 'failure': failure}) - publisher.close() + + with ConnectionPool.item() as conn: + publisher = DirectPublisher(connection=conn, msg_id=msg_id) + try: + publisher.send({'result': reply, 'failure': failure}) + except TypeError: + publisher.send( + {'result': dict((k, repr(v)) + for k, v in reply.__dict__.iteritems()), + 'failure': failure}) + + publisher.close() class RemoteError(exception.Error): @@ -347,8 +441,9 @@ def _unpack_context(msg): if key.startswith('_context_'): value = msg.pop(key) context_dict[key[9:]] = value + context_dict['msg_id'] = msg.pop('_msg_id', None) LOG.debug(_('unpacked context: %s'), context_dict) - return context.RequestContext.from_dict(context_dict) + return RpcContext.from_dict(context_dict) def _pack_context(msg, context): @@ -360,70 +455,112 @@ def _pack_context(msg, context): for args at some point. """ - context = dict([('_context_%s' % key, value) - for (key, value) in context.to_dict().iteritems()]) - msg.update(context) + context_d = dict([('_context_%s' % key, value) + for (key, value) in context.to_dict().iteritems()]) + msg.update(context_d) -def call(context, topic, msg): - """Sends a message on a topic and wait for a response.""" +class RpcContext(context.RequestContext): + def __init__(self, *args, **kwargs): + msg_id = kwargs.pop('msg_id', None) + self.msg_id = msg_id + super(RpcContext, self).__init__(*args, **kwargs) + + def reply(self, *args, **kwargs): + msg_reply(self.msg_id, *args, **kwargs) + + +def multicall(context, topic, msg): + """Make a call that returns multiple times.""" LOG.debug(_('Making asynchronous call on %s ...'), topic) msg_id = uuid.uuid4().hex msg.update({'_msg_id': msg_id}) LOG.debug(_('MSG_ID is %s') % (msg_id)) _pack_context(msg, context) - class WaitMessage(object): - def __call__(self, data, message): - """Acks message and sets result.""" - message.ack() - if data['failure']: - self.result = RemoteError(*data['failure']) - else: - self.result = data['result'] - - wait_msg = WaitMessage() - conn = Connection.instance() - consumer = DirectConsumer(connection=conn, msg_id=msg_id) + con_conn = ConnectionPool.get() + consumer = DirectConsumer(connection=con_conn, msg_id=msg_id) + wait_msg = MulticallWaiter(consumer) consumer.register_callback(wait_msg) - conn = Connection.instance() - publisher = TopicPublisher(connection=conn, topic=topic) + publisher = TopicPublisher(connection=con_conn, topic=topic) publisher.send(msg) publisher.close() - try: - consumer.wait(limit=1) - except StopIteration: - pass - consumer.close() - # NOTE(termie): this is a little bit of a change from the original - # non-eventlet code where returning a Failure - # instance from a deferred call is very similar to - # raising an exception - if isinstance(wait_msg.result, Exception): - raise wait_msg.result - return wait_msg.result + return wait_msg + + +class MulticallWaiter(object): + def __init__(self, consumer): + self._consumer = consumer + self._results = queue.Queue() + self._closed = False + + def close(self): + self._closed = True + self._consumer.close() + ConnectionPool.put(self._consumer.connection) + + def __call__(self, data, message): + """Acks message and sets result.""" + message.ack() + if data['failure']: + self._results.put(RemoteError(*data['failure'])) + else: + self._results.put(data['result']) + + def __iter__(self): + return self.wait() + + def wait(self): + while True: + rv = None + while rv is None and not self._closed: + try: + rv = self._consumer.fetch(enable_callbacks=True) + except Exception: + self.close() + raise + time.sleep(0.01) + + result = self._results.get() + if isinstance(result, Exception): + self.close() + raise result + if result == None: + self.close() + raise StopIteration + yield result + + +def call(context, topic, msg): + """Sends a message on a topic and wait for a response.""" + rv = multicall(context, topic, msg) + # NOTE(vish): return the last result from the multicall + rv = list(rv) + if not rv: + return + return rv[-1] def cast(context, topic, msg): """Sends a message on a topic without waiting for a response.""" LOG.debug(_('Making asynchronous cast on %s...'), topic) _pack_context(msg, context) - conn = Connection.instance() - publisher = TopicPublisher(connection=conn, topic=topic) - publisher.send(msg) - publisher.close() + with ConnectionPool.item() as conn: + publisher = TopicPublisher(connection=conn, topic=topic) + publisher.send(msg) + publisher.close() def fanout_cast(context, topic, msg): """Sends a message on a fanout exchange without waiting for a response.""" LOG.debug(_('Making asynchronous fanout cast...')) _pack_context(msg, context) - conn = Connection.instance() - publisher = FanoutPublisher(topic, connection=conn) - publisher.send(msg) - publisher.close() + with ConnectionPool.item() as conn: + publisher = FanoutPublisher(topic, connection=conn) + publisher.send(msg) + publisher.close() def generic_response(message_data, message): @@ -459,6 +596,7 @@ def send_message(topic, message, wait=True): if wait: consumer.wait() + consumer.close() if __name__ == '__main__': diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 55f8e0a6d..1bb047e2e 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -24,6 +24,7 @@ from nova import exception from nova import flags from nova import log as logging from nova import rpc +from nova import utils from eventlet import greenpool @@ -84,7 +85,7 @@ def get_zone_capabilities(context): def select(context, specs=None): """Returns a list of hosts.""" return _call_scheduler('select', context=context, - params={"specs": specs}) + params={"request_spec": specs}) def update_service_capabilities(context, service_name, host, capabilities): @@ -106,12 +107,14 @@ def _wrap_method(function, self): def _process(func, zone): """Worker stub for green thread pool. Give the worker an authenticated nova client and zone info.""" - nova = novaclient.OpenStack(zone.username, zone.password, zone.api_url) + nova = novaclient.OpenStack(zone.username, zone.password, None, + zone.api_url) nova.authenticate() return func(nova, zone) -def call_zone_method(context, method, errors_to_ignore=None, *args, **kwargs): +def call_zone_method(context, method_name, errors_to_ignore=None, + novaclient_collection_name='zones', *args, **kwargs): """Returns a list of (zone, call_result) objects.""" if not isinstance(errors_to_ignore, (list, tuple)): # This will also handle the default None @@ -121,7 +124,7 @@ def call_zone_method(context, method, errors_to_ignore=None, *args, **kwargs): results = [] for zone in db.zone_get_all(context): try: - nova = novaclient.OpenStack(zone.username, zone.password, + nova = novaclient.OpenStack(zone.username, zone.password, None, zone.api_url) nova.authenticate() except novaclient.exceptions.BadRequest, e: @@ -131,18 +134,16 @@ def call_zone_method(context, method, errors_to_ignore=None, *args, **kwargs): #TODO (dabo) - add logic for failure counts per zone, # with escalation after a given number of failures. continue - zone_method = getattr(nova.zones, method) + novaclient_collection = getattr(nova, novaclient_collection_name) + collection_method = getattr(novaclient_collection, method_name) def _error_trap(*args, **kwargs): try: - return zone_method(*args, **kwargs) + return collection_method(*args, **kwargs) except Exception as e: if type(e) in errors_to_ignore: return None - # TODO (dabo) - want to be able to re-raise here. - # Returning a string now; raising was causing issues. - # raise e - return "ERROR", "%s" % e + raise res = pool.spawn(_error_trap, *args, **kwargs) results.append((zone, res)) @@ -201,38 +202,78 @@ class RedirectResult(exception.Error): class reroute_compute(object): - """Decorator used to indicate that the method should - delegate the call the child zones if the db query - can't find anything.""" + """ + reroute_compute is responsible for trying to lookup a resource in the + current zone and if it's not found there, delegating the call to the + child zones. + + Since reroute_compute will be making 'cross-zone' calls, the ID for the + object must come in as a UUID-- if we receive an integer ID, we bail. + + The steps involved are: + + 1. Validate that item_id is UUID like + + 2. Lookup item by UUID in the zone local database + + 3. If the item was found, then extract integer ID, and pass that to + the wrapped method. (This ensures that zone-local code can + continue to use integer IDs). + + 4. If the item was not found, we delgate the call to a child zone + using the UUID. + """ def __init__(self, method_name): self.method_name = method_name + def _route_to_child_zones(self, context, collection, item_uuid): + if not FLAGS.enable_zone_routing: + raise exception.InstanceNotFound(instance_id=item_uuid) + + zones = db.zone_get_all(context) + if not zones: + raise exception.InstanceNotFound(instance_id=item_uuid) + + # Ask the children to provide an answer ... + LOG.debug(_("Asking child zones ...")) + result = self._call_child_zones(zones, + wrap_novaclient_function(_issue_novaclient_command, + collection, self.method_name, item_uuid)) + # Scrub the results and raise another exception + # so the API layers can bail out gracefully ... + raise RedirectResult(self.unmarshall_result(result)) + def __call__(self, f): def wrapped_f(*args, **kwargs): - collection, context, item_id = \ + collection, context, item_id_or_uuid = \ self.get_collection_context_and_id(args, kwargs) - try: - # Call the original function ... + + attempt_reroute = False + if utils.is_uuid_like(item_id_or_uuid): + item_uuid = item_id_or_uuid + try: + instance = db.instance_get_by_uuid(context, item_uuid) + except exception.InstanceNotFound, e: + # NOTE(sirp): since a UUID was passed in, we can attempt + # to reroute to a child zone + attempt_reroute = True + LOG.debug(_("Instance %(item_uuid)s not found " + "locally: '%(e)s'" % locals())) + else: + # NOTE(sirp): since we're not re-routing in this case, and + # we we were passed a UUID, we need to replace that UUID + # with an integer ID in the argument list so that the + # zone-local code can continue to use integer IDs. + item_id = instance['id'] + args = list(args) # needs to be mutable to replace + self.replace_uuid_with_id(args, kwargs, item_id) + + if attempt_reroute: + return self._route_to_child_zones(context, collection, + item_uuid) + else: return f(*args, **kwargs) - except exception.InstanceNotFound, e: - LOG.debug(_("Instance %(item_id)s not found " - "locally: '%(e)s'" % locals())) - - if not FLAGS.enable_zone_routing: - raise - - zones = db.zone_get_all(context) - if not zones: - raise - - # Ask the children to provide an answer ... - LOG.debug(_("Asking child zones ...")) - result = self._call_child_zones(zones, - wrap_novaclient_function(_issue_novaclient_command, - collection, self.method_name, item_id)) - # Scrub the results and raise another exception - # so the API layers can bail out gracefully ... - raise RedirectResult(self.unmarshall_result(result)) + return wrapped_f def _call_child_zones(self, zones, function): @@ -251,6 +292,18 @@ class reroute_compute(object): instance_id = args[2] return ("servers", context, instance_id) + @staticmethod + def replace_uuid_with_id(args, kwargs, replacement_id): + """ + Extracts the UUID parameter from the arg or kwarg list and replaces + it with an integer ID. + """ + if 'instance_id' in kwargs: + kwargs['instance_id'] = replacement_id + elif len(args) > 1: + args.pop(2) + args.insert(2, replacement_id) + def unmarshall_result(self, zone_responses): """Result is a list of responses from each child zone. Each decorator derivation is responsible to turning this diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index 2094e3565..0b257c5d8 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -28,6 +28,7 @@ from nova import exception from nova import flags from nova import log as logging from nova import rpc +from nova import utils from nova.compute import power_state FLAGS = flags.FLAGS @@ -61,7 +62,7 @@ class Scheduler(object): """Check whether a service is up based on last heartbeat.""" last_heartbeat = service['updated_at'] or service['created_at'] # Timestamps in DB are UTC. - elapsed = datetime.datetime.utcnow() - last_heartbeat + elapsed = utils.utcnow() - last_heartbeat return elapsed < datetime.timedelta(seconds=FLAGS.service_down_time) def hosts_up(self, context, topic): diff --git a/nova/scheduler/host_filter.py b/nova/scheduler/host_filter.py index 483f3225c..bd6b26608 100644 --- a/nova/scheduler/host_filter.py +++ b/nova/scheduler/host_filter.py @@ -14,8 +14,8 @@ # under the License. """ -Host Filter is a driver mechanism for requesting instance resources. -Three drivers are included: AllHosts, Flavor & JSON. AllHosts just +Host Filter is a mechanism for requesting instance resources. +Three filters are included: AllHosts, Flavor & JSON. AllHosts just returns the full, unfiltered list of hosts. Flavor is a hard coded matching mechanism based on flavor criteria and JSON is an ad-hoc filter grammar. @@ -41,18 +41,20 @@ import json from nova import exception from nova import flags from nova import log as logging +from nova.scheduler import zone_aware_scheduler from nova import utils +from nova.scheduler import zone_aware_scheduler LOG = logging.getLogger('nova.scheduler.host_filter') FLAGS = flags.FLAGS -flags.DEFINE_string('default_host_filter_driver', +flags.DEFINE_string('default_host_filter', 'nova.scheduler.host_filter.AllHostsFilter', - 'Which driver to use for filtering hosts.') + 'Which filter to use for filtering hosts.') class HostFilter(object): - """Base class for host filter drivers.""" + """Base class for host filters.""" def instance_type_to_filter(self, instance_type): """Convert instance_type into a filter for most common use-case.""" @@ -63,14 +65,15 @@ class HostFilter(object): raise NotImplementedError() def _full_name(self): - """module.classname of the filter driver""" + """module.classname of the filter.""" return "%s.%s" % (self.__module__, self.__class__.__name__) class AllHostsFilter(HostFilter): - """NOP host filter driver. Returns all hosts in ZoneManager. + """ NOP host filter. Returns all hosts in ZoneManager. This essentially does what the old Scheduler+Chance used - to give us.""" + to give us. + """ def instance_type_to_filter(self, instance_type): """Return anything to prevent base-class from raising @@ -83,8 +86,8 @@ class AllHostsFilter(HostFilter): for host, services in zone_manager.service_states.iteritems()] -class FlavorFilter(HostFilter): - """HostFilter driver hard-coded to work with flavors.""" +class InstanceTypeFilter(HostFilter): + """HostFilter hard-coded to work with InstanceType records.""" def instance_type_to_filter(self, instance_type): """Use instance_type to filter hosts.""" @@ -98,9 +101,10 @@ class FlavorFilter(HostFilter): capabilities = services.get('compute', {}) host_ram_mb = capabilities['host_memory_free'] disk_bytes = capabilities['disk_available'] - if host_ram_mb >= instance_type['memory_mb'] and \ - disk_bytes >= instance_type['local_gb']: - selected_hosts.append((host, capabilities)) + spec_ram = instance_type['memory_mb'] + spec_disk = instance_type['local_gb'] + if host_ram_mb >= spec_ram and disk_bytes >= spec_disk: + selected_hosts.append((host, capabilities)) return selected_hosts #host entries (currently) are like: @@ -109,15 +113,15 @@ class FlavorFilter(HostFilter): # 'host_memory_total': 8244539392, # 'host_memory_overhead': 184225792, # 'host_memory_free': 3868327936, -# 'host_memory_free_computed': 3840843776}, -# 'host_other-config': {}, +# 'host_memory_free_computed': 3840843776, +# 'host_other_config': {}, # 'host_ip_address': '192.168.1.109', # 'host_cpu_info': {}, # 'disk_available': 32954957824, # 'disk_total': 50394562560, -# 'disk_used': 17439604736}, +# 'disk_used': 17439604736, # 'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f', -# 'host_name-label': 'xs-mini'} +# 'host_name_label': 'xs-mini'} # instance_type table has: #name = Column(String(255), unique=True) @@ -131,8 +135,9 @@ class FlavorFilter(HostFilter): class JsonFilter(HostFilter): - """Host Filter driver to allow simple JSON-based grammar for - selecting hosts.""" + """Host Filter to allow simple JSON-based grammar for + selecting hosts. + """ def _equals(self, args): """First term is == all the other terms.""" @@ -222,13 +227,14 @@ class JsonFilter(HostFilter): required_disk = instance_type['local_gb'] query = ['and', ['>=', '$compute.host_memory_free', required_ram], - ['>=', '$compute.disk_available', required_disk] + ['>=', '$compute.disk_available', required_disk], ] return (self._full_name(), json.dumps(query)) def _parse_string(self, string, host, services): """Strings prefixed with $ are capability lookups in the - form '$service.capability[.subcap*]'""" + form '$service.capability[.subcap*]' + """ if not string: return None if string[0] != '$': @@ -271,18 +277,48 @@ class JsonFilter(HostFilter): return hosts -DRIVERS = [AllHostsFilter, FlavorFilter, JsonFilter] +FILTERS = [AllHostsFilter, InstanceTypeFilter, JsonFilter] -def choose_driver(driver_name=None): - """Since the caller may specify which driver to use we need - to have an authoritative list of what is permissible. This - function checks the driver name against a predefined set - of acceptable drivers.""" +def choose_host_filter(filter_name=None): + """Since the caller may specify which filter to use we need + to have an authoritative list of what is permissible. This + function checks the filter name against a predefined set + of acceptable filters. + """ - if not driver_name: - driver_name = FLAGS.default_host_filter_driver - for driver in DRIVERS: - if "%s.%s" % (driver.__module__, driver.__name__) == driver_name: - return driver() - raise exception.SchedulerHostFilterDriverNotFound(driver_name=driver_name) + if not filter_name: + filter_name = FLAGS.default_host_filter + for filter_class in FILTERS: + host_match = "%s.%s" % (filter_class.__module__, filter_class.__name__) + if host_match == filter_name: + return filter_class() + raise exception.SchedulerHostFilterNotFound(filter_name=filter_name) + + +class HostFilterScheduler(zone_aware_scheduler.ZoneAwareScheduler): + """The HostFilterScheduler uses the HostFilter to filter + hosts for weighing. The particular filter used may be passed in + as an argument or the default will be used. + + request_spec = {'filter': <Filter name>, + 'instance_type': <InstanceType dict>} + """ + + def filter_hosts(self, num, request_spec): + """Filter the full host list (from the ZoneManager)""" + filter_name = request_spec.get('filter', None) + host_filter = choose_host_filter(filter_name) + + # TODO(sandy): We're only using InstanceType-based specs + # currently. Later we'll need to snoop for more detailed + # host filter requests. + instance_type = request_spec['instance_type'] + name, query = host_filter.instance_type_to_filter(instance_type) + return host_filter.filter_hosts(self.zone_manager, query) + + def weigh_hosts(self, num, request_spec, hosts): + """Derived classes must override this method and return + a lists of hosts in [{weight, hostname}] format. + """ + return [dict(weight=1, hostname=host) for host, caps in hosts] diff --git a/nova/scheduler/least_cost.py b/nova/scheduler/least_cost.py new file mode 100644 index 000000000..629fe2e42 --- /dev/null +++ b/nova/scheduler/least_cost.py @@ -0,0 +1,156 @@ +# Copyright (c) 2011 Openstack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Least Cost Scheduler is a mechanism for choosing which host machines to +provision a set of resources to. The input of the least-cost-scheduler is a +set of objective-functions, called the 'cost-functions', a weight for each +cost-function, and a list of candidate hosts (gathered via FilterHosts). + +The cost-function and weights are tabulated, and the host with the least cost +is then selected for provisioning. +""" + +import collections + +from nova import flags +from nova import log as logging +from nova.scheduler import zone_aware_scheduler +from nova import utils + +LOG = logging.getLogger('nova.scheduler.least_cost') + +FLAGS = flags.FLAGS +flags.DEFINE_list('least_cost_scheduler_cost_functions', + ['nova.scheduler.least_cost.noop_cost_fn'], + 'Which cost functions the LeastCostScheduler should use.') + + +# TODO(sirp): Once we have enough of these rules, we can break them out into a +# cost_functions.py file (perhaps in a least_cost_scheduler directory) +flags.DEFINE_integer('noop_cost_fn_weight', 1, + 'How much weight to give the noop cost function') + + +def noop_cost_fn(host): + """Return a pre-weight cost of 1 for each host""" + return 1 + + +flags.DEFINE_integer('fill_first_cost_fn_weight', 1, + 'How much weight to give the fill-first cost function') + + +def fill_first_cost_fn(host): + """Prefer hosts that have less ram available, filter_hosts will exclude + hosts that don't have enough ram""" + hostname, caps = host + free_mem = caps['compute']['host_memory_free'] + return free_mem + + +class LeastCostScheduler(zone_aware_scheduler.ZoneAwareScheduler): + def get_cost_fns(self): + """Returns a list of tuples containing weights and cost functions to + use for weighing hosts + """ + cost_fns = [] + for cost_fn_str in FLAGS.least_cost_scheduler_cost_functions: + + try: + # NOTE(sirp): import_class is somewhat misnamed since it can + # any callable from a module + cost_fn = utils.import_class(cost_fn_str) + except exception.ClassNotFound: + raise exception.SchedulerCostFunctionNotFound( + cost_fn_str=cost_fn_str) + + try: + weight = getattr(FLAGS, "%s_weight" % cost_fn.__name__) + except AttributeError: + raise exception.SchedulerWeightFlagNotFound( + flag_name=flag_name) + + cost_fns.append((weight, cost_fn)) + + return cost_fns + + def weigh_hosts(self, num, request_spec, hosts): + """Returns a list of dictionaries of form: + [ {weight: weight, hostname: hostname} ]""" + + # FIXME(sirp): weigh_hosts should handle more than just instances + hostnames = [hostname for hostname, caps in hosts] + + cost_fns = self.get_cost_fns() + costs = weighted_sum(domain=hosts, weighted_fns=cost_fns) + + weighted = [] + weight_log = [] + for cost, hostname in zip(costs, hostnames): + weight_log.append("%s: %s" % (hostname, "%.2f" % cost)) + weight_dict = dict(weight=cost, hostname=hostname) + weighted.append(weight_dict) + + LOG.debug(_("Weighted Costs => %s") % weight_log) + return weighted + + +def normalize_list(L): + """Normalize an array of numbers such that each element satisfies: + 0 <= e <= 1""" + if not L: + return L + max_ = max(L) + if max_ > 0: + return [(float(e) / max_) for e in L] + return L + + +def weighted_sum(domain, weighted_fns, normalize=True): + """Use the weighted-sum method to compute a score for an array of objects. + Normalize the results of the objective-functions so that the weights are + meaningful regardless of objective-function's range. + + domain - input to be scored + weighted_fns - list of weights and functions like: + [(weight, objective-functions)] + + Returns an unsorted of scores. To pair with hosts do: zip(scores, hosts) + """ + # Table of form: + # { domain1: [score1, score2, ..., scoreM] + # ... + # domainN: [score1, score2, ..., scoreM] } + score_table = collections.defaultdict(list) + for weight, fn in weighted_fns: + scores = [fn(elem) for elem in domain] + + if normalize: + norm_scores = normalize_list(scores) + else: + norm_scores = scores + + for idx, score in enumerate(norm_scores): + weighted_score = score * weight + score_table[idx].append(weighted_score) + + # Sum rows in table to compute score for each element in domain + domain_scores = [] + for idx in sorted(score_table): + elem_score = sum(score_table[idx]) + elem = domain[idx] + domain_scores.append(elem_score) + + return domain_scores diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index 55cd7208b..6cb75aa8d 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -70,6 +70,14 @@ class SchedulerManager(manager.Manager): self.zone_manager.update_service_capabilities(service_name, host, capabilities) + def select(self, context=None, *args, **kwargs): + """Select a list of hosts best matching the provided specs.""" + return self.driver.select(context, *args, **kwargs) + + def get_scheduler_rules(self, context=None, *args, **kwargs): + """Ask the driver how requests should be made of it.""" + return self.driver.get_scheduler_rules(context, *args, **kwargs) + def _schedule(self, method, context, topic, *args, **kwargs): """Tries to call schedule_* method on the driver to retrieve host. @@ -80,14 +88,21 @@ class SchedulerManager(manager.Manager): try: host = getattr(self.driver, driver_method)(elevated, *args, **kwargs) - except AttributeError: + except AttributeError, e: + LOG.warning(_("Driver Method %(driver_method)s missing: %(e)s." + "Reverting to schedule()") % locals()) host = self.driver.schedule(elevated, topic, *args, **kwargs) + if not host: + LOG.debug(_("%(topic)s %(method)s handled in Scheduler") + % locals()) + return + rpc.cast(context, db.queue_get_for(context, topic, host), {"method": method, "args": kwargs}) - LOG.debug(_("Casting to %(topic)s %(host)s for %(method)s") % locals()) + LOG.debug(_("Casted to %(topic)s %(host)s for %(method)s") % locals()) # NOTE (masumotok) : This method should be moved to nova.api.ec2.admin. # Based on bexar design summit discussion, diff --git a/nova/scheduler/simple.py b/nova/scheduler/simple.py index dd568d2c6..fc1b3142a 100644 --- a/nova/scheduler/simple.py +++ b/nova/scheduler/simple.py @@ -21,10 +21,9 @@ Simple Scheduler """ -import datetime - from nova import db from nova import flags +from nova import utils from nova.scheduler import driver from nova.scheduler import chance @@ -40,7 +39,7 @@ flags.DEFINE_integer("max_networks", 1000, class SimpleScheduler(chance.ChanceScheduler): """Implements Naive Scheduler that tries to find least loaded host.""" - def schedule_run_instance(self, context, instance_id, *_args, **_kwargs): + def _schedule_instance(self, context, instance_id, *_args, **_kwargs): """Picks a host that is up and has the fewest running instances.""" instance_ref = db.instance_get(context, instance_id) if (instance_ref['availability_zone'] @@ -54,7 +53,7 @@ class SimpleScheduler(chance.ChanceScheduler): # TODO(vish): this probably belongs in the manager, if we # can generalize this somehow - now = datetime.datetime.utcnow() + now = utils.utcnow() db.instance_update(context, instance_id, {'host': host, 'scheduled_at': now}) return host @@ -66,7 +65,7 @@ class SimpleScheduler(chance.ChanceScheduler): if self.service_is_up(service): # NOTE(vish): this probably belongs in the manager, if we # can generalize this somehow - now = datetime.datetime.utcnow() + now = utils.utcnow() db.instance_update(context, instance_id, {'host': service['host'], @@ -76,6 +75,12 @@ class SimpleScheduler(chance.ChanceScheduler): " for this request. Is the appropriate" " service running?")) + def schedule_run_instance(self, context, instance_id, *_args, **_kwargs): + return self._schedule_instance(context, instance_id, *_args, **_kwargs) + + def schedule_start_instance(self, context, instance_id, *_args, **_kwargs): + return self._schedule_instance(context, instance_id, *_args, **_kwargs) + def schedule_create_volume(self, context, volume_id, *_args, **_kwargs): """Picks a host that is up and has the fewest volumes.""" volume_ref = db.volume_get(context, volume_id) @@ -90,7 +95,7 @@ class SimpleScheduler(chance.ChanceScheduler): # TODO(vish): this probably belongs in the manager, if we # can generalize this somehow - now = datetime.datetime.utcnow() + now = utils.utcnow() db.volume_update(context, volume_id, {'host': host, 'scheduled_at': now}) return host @@ -103,7 +108,7 @@ class SimpleScheduler(chance.ChanceScheduler): if self.service_is_up(service): # NOTE(vish): this probably belongs in the manager, if we # can generalize this somehow - now = datetime.datetime.utcnow() + now = utils.utcnow() db.volume_update(context, volume_id, {'host': service['host'], diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index b3d230bd2..e7bff2faa 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -21,14 +21,30 @@ across zones. There are two expansion points to this class for: """ import operator +import json +import M2Crypto +import novaclient + +from nova import crypto +from nova import db +from nova import exception +from nova import flags from nova import log as logging +from nova import rpc + from nova.scheduler import api from nova.scheduler import driver +FLAGS = flags.FLAGS LOG = logging.getLogger('nova.scheduler.zone_aware_scheduler') +class InvalidBlob(exception.NovaException): + message = _("Ill-formed or incorrectly routed 'blob' data sent " + "to instance create request.") + + class ZoneAwareScheduler(driver.Scheduler): """Base class for creating Zone Aware Schedulers.""" @@ -36,84 +52,220 @@ class ZoneAwareScheduler(driver.Scheduler): """Call novaclient zone method. Broken out for testing.""" return api.call_zone_method(context, method, specs=specs) - def schedule_run_instance(self, context, topic='compute', specs={}, - *args, **kwargs): + def _provision_resource_locally(self, context, item, instance_id, kwargs): + """Create the requested resource in this Zone.""" + host = item['hostname'] + kwargs['instance_id'] = instance_id + rpc.cast(context, + db.queue_get_for(context, "compute", host), + {"method": "run_instance", + "args": kwargs}) + LOG.debug(_("Provisioning locally via compute node %(host)s") + % locals()) + + def _decrypt_blob(self, blob): + """Returns the decrypted blob or None if invalid. Broken out + for testing.""" + decryptor = crypto.decryptor(FLAGS.build_plan_encryption_key) + try: + json_entry = decryptor(blob) + return json.dumps(entry) + except M2Crypto.EVP.EVPError: + pass + return None + + def _ask_child_zone_to_create_instance(self, context, zone_info, + request_spec, kwargs): + """Once we have determined that the request should go to one + of our children, we need to fabricate a new POST /servers/ + call with the same parameters that were passed into us. + + Note that we have to reverse engineer from our args to get back the + image, flavor, ipgroup, etc. since the original call could have + come in from EC2 (which doesn't use these things).""" + + instance_type = request_spec['instance_type'] + instance_properties = request_spec['instance_properties'] + + name = instance_properties['display_name'] + image_ref = instance_properties['image_ref'] + meta = instance_properties['metadata'] + flavor_id = instance_type['flavorid'] + reservation_id = instance_properties['reservation_id'] + + files = kwargs['injected_files'] + ipgroup = None # Not supported in OS API ... yet + + child_zone = zone_info['child_zone'] + child_blob = zone_info['child_blob'] + zone = db.zone_get(context, child_zone) + url = zone.api_url + LOG.debug(_("Forwarding instance create call to child zone %(url)s" + ". ReservationID=%(reservation_id)s") + % locals()) + nova = None + try: + nova = novaclient.OpenStack(zone.username, zone.password, None, + url) + nova.authenticate() + except novaclient.exceptions.BadRequest, e: + raise exception.NotAuthorized(_("Bad credentials attempting " + "to talk to zone at %(url)s.") % locals()) + + nova.servers.create(name, image_ref, flavor_id, ipgroup, meta, files, + child_blob, reservation_id=reservation_id) + + def _provision_resource_from_blob(self, context, item, instance_id, + request_spec, kwargs): + """Create the requested resource locally or in a child zone + based on what is stored in the zone blob info. + + Attempt to decrypt the blob to see if this request is: + 1. valid, and + 2. intended for this zone or a child zone. + + Note: If we have "blob" that means the request was passed + into us from a parent zone. If we have "child_blob" that + means we gathered the info from one of our children. + It's possible that, when we decrypt the 'blob' field, it + contains "child_blob" data. In which case we forward the + request.""" + + host_info = None + if "blob" in item: + # Request was passed in from above. Is it for us? + host_info = self._decrypt_blob(item['blob']) + elif "child_blob" in item: + # Our immediate child zone provided this info ... + host_info = item + + if not host_info: + raise InvalidBlob() + + # Valid data ... is it for us? + if 'child_zone' in host_info and 'child_blob' in host_info: + self._ask_child_zone_to_create_instance(context, host_info, + request_spec, kwargs) + else: + self._provision_resource_locally(context, host_info, + instance_id, kwargs) + + def _provision_resource(self, context, item, instance_id, request_spec, + kwargs): + """Create the requested resource in this Zone or a child zone.""" + if "hostname" in item: + self._provision_resource_locally(context, item, instance_id, + kwargs) + return + + self._provision_resource_from_blob(context, item, instance_id, + request_spec, kwargs) + + def schedule_run_instance(self, context, instance_id, request_spec, + *args, **kwargs): """This method is called from nova.compute.api to provision an instance. However we need to look at the parameters being passed in to see if this is a request to: 1. Create a Build Plan and then provision, or 2. Use the Build Plan information in the request parameters to simply create the instance (either in this zone or - a child zone).""" + a child zone). + """ + + # TODO(sandy): We'll have to look for richer specs at some point. - if 'blob' in specs: - return self.provision_instance(context, topic, specs) + blob = request_spec.get('blob') + if blob: + self._provision_resource(context, request_spec, instance_id, + request_spec, kwargs) + return None # Create build plan and provision ... - build_plan = self.select(context, specs) - for item in build_plan: - self.provision_instance(context, topic, item) + build_plan = self.select(context, request_spec) + if not build_plan: + raise driver.NoValidHost(_('No hosts were available')) + + for num in xrange(request_spec['num_instances']): + if not build_plan: + break - def provision_instance(context, topic, item): - """Create the requested instance in this Zone or a child zone.""" - pass + item = build_plan.pop(0) + self._provision_resource(context, item, instance_id, request_spec, + kwargs) - def select(self, context, *args, **kwargs): + # Returning None short-circuits the routing to Compute (since + # we've already done it here) + return None + + def select(self, context, request_spec, *args, **kwargs): """Select returns a list of weights and zone/host information corresponding to the best hosts to service the request. Any child zone information has been encrypted so as not to reveal - anything about the children.""" - return self._schedule(context, "compute", *args, **kwargs) + anything about the children. + """ + return self._schedule(context, "compute", request_spec, + *args, **kwargs) - def schedule(self, context, topic, *args, **kwargs): + # TODO(sandy): We're only focused on compute instances right now, + # so we don't implement the default "schedule()" method required + # of Schedulers. + def schedule(self, context, topic, request_spec, *args, **kwargs): """The schedule() contract requires we return the one best-suited host for this request. """ - res = self._schedule(context, topic, *args, **kwargs) - # TODO(sirp): should this be a host object rather than a weight-dict? - if not res: - raise driver.NoValidHost(_('No hosts were available')) - return res[0] + raise driver.NoValidHost(_('No hosts were available')) - def _schedule(self, context, topic, *args, **kwargs): + def _schedule(self, context, topic, request_spec, *args, **kwargs): """Returns a list of hosts that meet the required specs, ordered by their fitness. """ - #TODO(sandy): extract these from args. + if topic != "compute": + raise NotImplemented(_("Zone Aware Scheduler only understands " + "Compute nodes (for now)")) + + #TODO(sandy): how to infer this from OS API params? num_instances = 1 - specs = {} # Filter local hosts based on requirements ... - host_list = self.filter_hosts(num_instances, specs) + host_list = self.filter_hosts(num_instances, request_spec) + + # TODO(sirp): weigh_hosts should also be a function of 'topic' or + # resources, so that we can apply different objective functions to it # then weigh the selected hosts. # weighted = [{weight=weight, name=hostname}, ...] - weighted = self.weigh_hosts(num_instances, specs, host_list) + weighted = self.weigh_hosts(num_instances, request_spec, host_list) # Next, tack on the best weights from the child zones ... + json_spec = json.dumps(request_spec) child_results = self._call_zone_method(context, "select", - specs=specs) + specs=json_spec) for child_zone, result in child_results: for weighting in result: # Remember the child_zone so we can get back to # it later if needed. This implicitly builds a zone # path structure. - host_dict = { - "weight": weighting["weight"], - "child_zone": child_zone, - "child_blob": weighting["blob"]} + host_dict = {"weight": weighting["weight"], + "child_zone": child_zone, + "child_blob": weighting["blob"]} weighted.append(host_dict) weighted.sort(key=operator.itemgetter('weight')) return weighted - def filter_hosts(self, num, specs): + def filter_hosts(self, num, request_spec): """Derived classes must override this method and return - a list of hosts in [(hostname, capability_dict)] format.""" - raise NotImplemented() + a list of hosts in [(hostname, capability_dict)] format. + """ + # NOTE(sirp): The default logic is the equivalent to AllHostsFilter + service_states = self.zone_manager.service_states + return [(host, services) + for host, services in service_states.iteritems()] - def weigh_hosts(self, num, specs, hosts): - """Derived classes must override this method and return - a lists of hosts in [{weight, hostname}] format.""" - raise NotImplemented() + def weigh_hosts(self, num, request_spec, hosts): + """Derived classes may override this to provide more sophisticated + scheduling objectives + """ + # NOTE(sirp): The default logic is the same as the NoopCostFunction + return [dict(weight=1, hostname=host) for host, caps in hosts] diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index 3ddf6f3c3..ba7403c15 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -17,16 +17,17 @@ ZoneManager oversees all communications with child Zones. """ +import datetime import novaclient import thread import traceback -from datetime import datetime from eventlet import greenpool from nova import db from nova import flags from nova import log as logging +from nova import utils FLAGS = flags.FLAGS flags.DEFINE_integer('zone_db_check_interval', 60, @@ -42,7 +43,7 @@ class ZoneState(object): self.name = None self.capabilities = None self.attempt = 0 - self.last_seen = datetime.min + self.last_seen = datetime.datetime.min self.last_exception = None self.last_exception_time = None @@ -56,7 +57,7 @@ class ZoneState(object): def update_metadata(self, zone_metadata): """Update zone metadata after successful communications with child zone.""" - self.last_seen = datetime.now() + self.last_seen = utils.utcnow() self.attempt = 0 self.name = zone_metadata.get("name", "n/a") self.capabilities = ", ".join(["%s=%s" % (k, v) @@ -72,7 +73,7 @@ class ZoneState(object): """Something went wrong. Check to see if zone should be marked as offline.""" self.last_exception = exception - self.last_exception_time = datetime.now() + self.last_exception_time = utils.utcnow() api_url = self.api_url logging.warning(_("'%(exception)s' error talking to " "zone %(api_url)s") % locals()) @@ -88,7 +89,8 @@ class ZoneState(object): def _call_novaclient(zone): """Call novaclient. Broken out for testing purposes.""" - client = novaclient.OpenStack(zone.username, zone.password, zone.api_url) + client = novaclient.OpenStack(zone.username, zone.password, None, + zone.api_url) return client.zones.info()._info @@ -104,7 +106,7 @@ def _poll_zone(zone): class ZoneManager(object): """Keeps the zone states updated.""" def __init__(self): - self.last_zone_db_check = datetime.min + self.last_zone_db_check = datetime.datetime.min self.zone_states = {} # { <zone_id> : ZoneState } self.service_states = {} # { <host> : { <service> : { cap k : v }}} self.green_pool = greenpool.GreenPool() @@ -158,10 +160,10 @@ class ZoneManager(object): def ping(self, context=None): """Ping should be called periodically to update zone status.""" - diff = datetime.now() - self.last_zone_db_check + diff = utils.utcnow() - self.last_zone_db_check if diff.seconds >= FLAGS.zone_db_check_interval: logging.debug(_("Updating zone cache from db.")) - self.last_zone_db_check = datetime.now() + self.last_zone_db_check = utils.utcnow() self._refresh_from_db(context) self._poll_zones(context) diff --git a/nova/service.py b/nova/service.py index ab1238c3b..74f9f04d8 100644 --- a/nova/service.py +++ b/nova/service.py @@ -19,14 +19,11 @@ """Generic Node baseclass for all workers that run on hosts.""" +import greenlet import inspect import os -import sys -import time -from eventlet import event from eventlet import greenthread -from eventlet import greenpool from nova import context from nova import db @@ -91,27 +88,37 @@ class Service(object): if 'nova-compute' == self.binary: self.manager.update_available_resource(ctxt) - conn1 = rpc.Connection.instance(new=True) - conn2 = rpc.Connection.instance(new=True) - conn3 = rpc.Connection.instance(new=True) - if self.report_interval: - consumer_all = rpc.TopicAdapterConsumer( - connection=conn1, - topic=self.topic, - proxy=self) - consumer_node = rpc.TopicAdapterConsumer( - connection=conn2, - topic='%s.%s' % (self.topic, self.host), - proxy=self) - fanout = rpc.FanoutAdapterConsumer( - connection=conn3, - topic=self.topic, - proxy=self) - - self.timers.append(consumer_all.attach_to_eventlet()) - self.timers.append(consumer_node.attach_to_eventlet()) - self.timers.append(fanout.attach_to_eventlet()) + self.conn = rpc.Connection.instance(new=True) + logging.debug("Creating Consumer connection for Service %s" % + self.topic) + + # Share this same connection for these Consumers + consumer_all = rpc.TopicAdapterConsumer( + connection=self.conn, + topic=self.topic, + proxy=self) + consumer_node = rpc.TopicAdapterConsumer( + connection=self.conn, + topic='%s.%s' % (self.topic, self.host), + proxy=self) + fanout = rpc.FanoutAdapterConsumer( + connection=self.conn, + topic=self.topic, + proxy=self) + consumer_set = rpc.ConsumerSet( + connection=self.conn, + consumer_list=[consumer_all, consumer_node, fanout]) + + # Wait forever, processing these consumers + def _wait(): + try: + consumer_set.wait() + finally: + consumer_set.close() + + self.consumer_set_thread = greenthread.spawn(_wait) + if self.report_interval: pulse = utils.LoopingCall(self.report_state) pulse.start(interval=self.report_interval, now=False) self.timers.append(pulse) @@ -174,6 +181,11 @@ class Service(object): logging.warn(_('Service killed that has no database entry')) def stop(self): + self.consumer_set_thread.kill() + try: + self.consumer_set_thread.wait() + except greenlet.GreenletExit: + pass for x in self.timers: try: x.stop() diff --git a/nova/test.py b/nova/test.py index 4deb2a175..4a0a18fe7 100644 --- a/nova/test.py +++ b/nova/test.py @@ -23,7 +23,6 @@ inline callbacks. """ -import datetime import functools import os import shutil @@ -31,17 +30,16 @@ import uuid import unittest import mox -import shutil import stubout from eventlet import greenthread -from nova import context -from nova import db from nova import fakerabbit from nova import flags from nova import rpc +from nova import utils from nova import service from nova import wsgi +from nova.virt import fake FLAGS = flags.FLAGS @@ -71,7 +69,7 @@ class TestCase(unittest.TestCase): # NOTE(vish): We need a better method for creating fixtures for tests # now that we have some required db setup for the system # to work properly. - self.start = datetime.datetime.utcnow() + self.start = utils.utcnow() shutil.copyfile(os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db), os.path.join(FLAGS.state_path, FLAGS.sqlite_db)) @@ -85,6 +83,7 @@ class TestCase(unittest.TestCase): self._monkey_patch_attach() self._monkey_patch_wsgi() self._original_flags = FLAGS.FlagValuesDict() + rpc.ConnectionPool = rpc.Pool(max_size=FLAGS.rpc_conn_pool_size) def tearDown(self): """Runs after each test method to tear down test environment.""" @@ -99,6 +98,10 @@ class TestCase(unittest.TestCase): if FLAGS.fake_rabbit: fakerabbit.reset_all() + if FLAGS.connection_type == 'fake': + if hasattr(fake.FakeConnection, '_instance'): + del fake.FakeConnection._instance + # Reset any overriden flags self.reset_flags() @@ -181,7 +184,7 @@ class TestCase(unittest.TestCase): wsgi.Server.start = _wrapped_start # Useful assertions - def assertDictMatch(self, d1, d2): + def assertDictMatch(self, d1, d2, approx_equal=False, tolerance=0.001): """Assert two dicts are equivalent. This is a 'deep' match in the sense that it handles nested @@ -212,15 +215,26 @@ class TestCase(unittest.TestCase): for key in d1keys: d1value = d1[key] d2value = d2[key] + try: + error = abs(float(d1value) - float(d2value)) + within_tolerance = error <= tolerance + except (ValueError, TypeError): + # If both values aren't convertable to float, just ignore + # ValueError if arg is a str, TypeError if it's something else + # (like None) + within_tolerance = False + if hasattr(d1value, 'keys') and hasattr(d2value, 'keys'): self.assertDictMatch(d1value, d2value) elif 'DONTCARE' in (d1value, d2value): continue + elif approx_equal and within_tolerance: + continue elif d1value != d2value: raise_assertion("d1['%(key)s']=%(d1value)s != " "d2['%(key)s']=%(d2value)s" % locals()) - def assertDictListMatch(self, L1, L2): + def assertDictListMatch(self, L1, L2, approx_equal=False, tolerance=0.001): """Assert a list of dicts are equivalent.""" def raise_assertion(msg): L1str = str(L1) @@ -236,4 +250,5 @@ class TestCase(unittest.TestCase): 'len(L2)=%(L2count)d' % locals()) for d1, d2 in zip(L1, L2): - self.assertDictMatch(d1, d2) + self.assertDictMatch(d1, d2, approx_equal=approx_equal, + tolerance=tolerance) diff --git a/nova/tests/api/openstack/extensions/foxinsocks.py b/nova/tests/api/openstack/extensions/foxinsocks.py index dbdd0928a..03aad007a 100644 --- a/nova/tests/api/openstack/extensions/foxinsocks.py +++ b/nova/tests/api/openstack/extensions/foxinsocks.py @@ -17,12 +17,10 @@ import json -from nova import wsgi - from nova.api.openstack import extensions -class FoxInSocksController(wsgi.Controller): +class FoxInSocksController(object): def index(self, req): return "Try to say this Mr. Knox, sir..." diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index bf51239e6..c74974b16 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -16,7 +16,6 @@ # under the License. import copy -import datetime import json import random import string @@ -38,8 +37,8 @@ from nova.api.openstack import auth from nova.api.openstack import versions from nova.api.openstack import limits from nova.auth.manager import User, Project +import nova.image.fake from nova.image import glance -from nova.image import local from nova.image import service from nova.tests import fake_flags from nova.wsgi import Router @@ -104,10 +103,12 @@ def stub_out_key_pair_funcs(stubs, have_key_pair=True): def stub_out_image_service(stubs): - def fake_image_show(meh, context, id): - return dict(kernelId=1, ramdiskId=1) - - stubs.Set(local.LocalImageService, 'show', fake_image_show) + def fake_get_image_service(image_href): + image_id = int(str(image_href).split('/')[-1]) + return (nova.image.fake.FakeImageService(), image_id) + stubs.Set(nova.image, 'get_image_service', fake_get_image_service) + stubs.Set(nova.image, 'get_default_image_service', + lambda: nova.image.fake.FakeImageService()) def stub_out_auth(stubs): @@ -139,8 +140,10 @@ def stub_out_networking(stubs): def stub_out_compute_api_snapshot(stubs): - def snapshot(self, context, instance_id, name): - return 123 + def snapshot(self, context, instance_id, name, extra_properties=None): + props = dict(instance_id=instance_id, instance_ref=instance_id) + props.update(extra_properties or {}) + return dict(id='123', status='ACTIVE', name=name, properties=props) stubs.Set(nova.compute.API, 'snapshot', snapshot) @@ -166,12 +169,34 @@ def stub_out_glance(stubs, initial_fixtures=None): def __init__(self, initial_fixtures): self.fixtures = initial_fixtures or [] - def fake_get_images(self): + def _filter_images(self, filters=None, marker=None, limit=None): + found = True + if marker: + found = False + if limit == 0: + limit = None + + fixtures = [] + count = 0 + for f in self.fixtures: + if limit and count >= limit: + break + if found: + fixtures.append(f) + count = count + 1 + if f['id'] == marker: + found = True + + return fixtures + + def fake_get_images(self, filters=None, marker=None, limit=None): + fixtures = self._filter_images(filters, marker, limit) return [dict(id=f['id'], name=f['name']) - for f in self.fixtures] + for f in fixtures] - def fake_get_images_detailed(self): - return copy.deepcopy(self.fixtures) + def fake_get_images_detailed(self, filters=None, + marker=None, limit=None): + return self._filter_images(filters, marker, limit) def fake_get_image_meta(self, image_id): image = self._find_image(image_id) @@ -208,7 +233,7 @@ def stub_out_glance(stubs, initial_fixtures=None): def _find_image(self, image_id): for f in self.fixtures: - if f['id'] == image_id: + if str(f['id']) == str(image_id): return f return None @@ -253,7 +278,7 @@ class FakeAuthDatabase(object): @staticmethod def auth_token_create(context, token): - fake_token = FakeToken(created_at=datetime.datetime.now(), **token) + fake_token = FakeToken(created_at=utils.utcnow(), **token) FakeAuthDatabase.data[fake_token.token_hash] = fake_token FakeAuthDatabase.data['id_%i' % fake_token.id] = fake_token return fake_token @@ -328,6 +353,11 @@ class FakeAuthManager(object): return user.admin def is_project_member(self, user, project): + if not isinstance(project, Project): + try: + project = self.get_project(project) + except exc.NotFound: + raise webob.exc.HTTPUnauthorized() return ((user.id in project.member_ids) or (user.id == project.project_manager_id)) diff --git a/nova/tests/api/openstack/test_api.py b/nova/tests/api/openstack/test_api.py index c63431a45..7321c329f 100644 --- a/nova/tests/api/openstack/test_api.py +++ b/nova/tests/api/openstack/test_api.py @@ -15,6 +15,8 @@ # License for the specific language governing permissions and limitations # under the License. +import json + import webob.exc import webob.dec @@ -23,6 +25,7 @@ from webob import Request from nova import test from nova.api import openstack from nova.api.openstack import faults +from nova.tests.api.openstack import fakes class APITest(test.TestCase): @@ -31,6 +34,24 @@ class APITest(test.TestCase): # simpler version of the app than fakes.wsgi_app return openstack.FaultWrapper(inner_app) + def test_malformed_json(self): + req = webob.Request.blank('/') + req.method = 'POST' + req.body = '{' + req.headers["content-type"] = "application/json" + + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 400) + + def test_malformed_xml(self): + req = webob.Request.blank('/') + req.method = 'POST' + req.body = '<hi im not xml>' + req.headers["content-type"] = "application/xml" + + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 400) + def test_exceptions_are_converted_to_faults(self): @webob.dec.wsgify diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 8f189c744..af3478c7d 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -114,6 +114,28 @@ class Test(test.TestCase): self.assertEqual(result.status, '401 Unauthorized') self.assertEqual(self.destroy_called, True) + def test_authorize_project(self): + f = fakes.FakeAuthManager() + user = nova.auth.manager.User('id1', 'user1', 'user1_key', None, None) + f.add_user(user) + f.create_project('user1_project', user) + f.create_project('user2_project', user) + + req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) + req.headers['X-Auth-User'] = 'user1' + req.headers['X-Auth-Key'] = 'user1_key' + result = req.get_response(fakes.wsgi_app()) + self.assertEqual(result.status, '204 No Content') + + token = result.headers['X-Auth-Token'] + self.stubs.Set(nova.api.openstack, 'APIRouterV10', fakes.FakeRouter) + req = webob.Request.blank('/v1.0/fake') + req.headers['X-Auth-Token'] = token + req.headers['X-Auth-Project-Id'] = 'user2_project' + result = req.get_response(fakes.wsgi_app()) + self.assertEqual(result.status, '200 OK') + self.assertEqual(result.headers['X-Test-Success'], 'True') + def test_bad_user_bad_key(self): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'unknown_user' @@ -143,6 +165,49 @@ class Test(test.TestCase): result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') + def test_bad_project(self): + f = fakes.FakeAuthManager() + user1 = nova.auth.manager.User('id1', 'user1', 'user1_key', None, None) + user2 = nova.auth.manager.User('id2', 'user2', 'user2_key', None, None) + f.add_user(user1) + f.add_user(user2) + f.create_project('user1_project', user1) + f.create_project('user2_project', user2) + + req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) + req.headers['X-Auth-User'] = 'user1' + req.headers['X-Auth-Key'] = 'user1_key' + result = req.get_response(fakes.wsgi_app()) + self.assertEqual(result.status, '204 No Content') + + token = result.headers['X-Auth-Token'] + self.stubs.Set(nova.api.openstack, 'APIRouterV10', fakes.FakeRouter) + req = webob.Request.blank('/v1.0/fake') + req.headers['X-Auth-Token'] = token + req.headers['X-Auth-Project-Id'] = 'user2_project' + result = req.get_response(fakes.wsgi_app()) + self.assertEqual(result.status, '401 Unauthorized') + + def test_not_existing_project(self): + f = fakes.FakeAuthManager() + user1 = nova.auth.manager.User('id1', 'user1', 'user1_key', None, None) + f.add_user(user1) + f.create_project('user1_project', user1) + + req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) + req.headers['X-Auth-User'] = 'user1' + req.headers['X-Auth-Key'] = 'user1_key' + result = req.get_response(fakes.wsgi_app()) + self.assertEqual(result.status, '204 No Content') + + token = result.headers['X-Auth-Token'] + self.stubs.Set(nova.api.openstack, 'APIRouterV10', fakes.FakeRouter) + req = webob.Request.blank('/v1.0/fake') + req.headers['X-Auth-Token'] = token + req.headers['X-Auth-Project-Id'] = 'unknown_project' + result = req.get_response(fakes.wsgi_app()) + self.assertEqual(result.status, '401 Unauthorized') + class TestFunctional(test.TestCase): def test_token_expiry(self): diff --git a/nova/tests/api/openstack/test_common.py b/nova/tests/api/openstack/test_common.py index 8f57c5b67..9a9d9125c 100644 --- a/nova/tests/api/openstack/test_common.py +++ b/nova/tests/api/openstack/test_common.py @@ -24,7 +24,7 @@ import webob.exc from webob import Request from nova import test -from nova.api.openstack.common import limited +from nova.api.openstack import common class LimiterTest(test.TestCase): @@ -35,9 +35,7 @@ class LimiterTest(test.TestCase): """ def setUp(self): - """ - Run before each test. - """ + """ Run before each test. """ super(LimiterTest, self).setUp() self.tiny = range(1) self.small = range(10) @@ -45,127 +43,144 @@ class LimiterTest(test.TestCase): self.large = range(10000) def test_limiter_offset_zero(self): - """ - Test offset key works with 0. - """ + """ Test offset key works with 0. """ req = Request.blank('/?offset=0') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium) - self.assertEqual(limited(self.large, req), self.large[:1000]) + self.assertEqual(common.limited(self.tiny, req), self.tiny) + self.assertEqual(common.limited(self.small, req), self.small) + self.assertEqual(common.limited(self.medium, req), self.medium) + self.assertEqual(common.limited(self.large, req), self.large[:1000]) def test_limiter_offset_medium(self): - """ - Test offset key works with a medium sized number. - """ + """ Test offset key works with a medium sized number. """ req = Request.blank('/?offset=10') - self.assertEqual(limited(self.tiny, req), []) - self.assertEqual(limited(self.small, req), self.small[10:]) - self.assertEqual(limited(self.medium, req), self.medium[10:]) - self.assertEqual(limited(self.large, req), self.large[10:1010]) + self.assertEqual(common.limited(self.tiny, req), []) + self.assertEqual(common.limited(self.small, req), self.small[10:]) + self.assertEqual(common.limited(self.medium, req), self.medium[10:]) + self.assertEqual(common.limited(self.large, req), self.large[10:1010]) def test_limiter_offset_over_max(self): - """ - Test offset key works with a number over 1000 (max_limit). - """ + """ Test offset key works with a number over 1000 (max_limit). """ req = Request.blank('/?offset=1001') - self.assertEqual(limited(self.tiny, req), []) - self.assertEqual(limited(self.small, req), []) - self.assertEqual(limited(self.medium, req), []) - self.assertEqual(limited(self.large, req), self.large[1001:2001]) + self.assertEqual(common.limited(self.tiny, req), []) + self.assertEqual(common.limited(self.small, req), []) + self.assertEqual(common.limited(self.medium, req), []) + self.assertEqual( + common.limited(self.large, req), self.large[1001:2001]) def test_limiter_offset_blank(self): - """ - Test offset key works with a blank offset. - """ + """ Test offset key works with a blank offset. """ req = Request.blank('/?offset=') - self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) + self.assertRaises( + webob.exc.HTTPBadRequest, common.limited, self.tiny, req) def test_limiter_offset_bad(self): - """ - Test offset key works with a BAD offset. - """ + """ Test offset key works with a BAD offset. """ req = Request.blank(u'/?offset=\u0020aa') - self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) + self.assertRaises( + webob.exc.HTTPBadRequest, common.limited, self.tiny, req) def test_limiter_nothing(self): - """ - Test request with no offset or limit - """ + """ Test request with no offset or limit """ req = Request.blank('/') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium) - self.assertEqual(limited(self.large, req), self.large[:1000]) + self.assertEqual(common.limited(self.tiny, req), self.tiny) + self.assertEqual(common.limited(self.small, req), self.small) + self.assertEqual(common.limited(self.medium, req), self.medium) + self.assertEqual(common.limited(self.large, req), self.large[:1000]) def test_limiter_limit_zero(self): - """ - Test limit of zero. - """ + """ Test limit of zero. """ req = Request.blank('/?limit=0') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium) - self.assertEqual(limited(self.large, req), self.large[:1000]) + self.assertEqual(common.limited(self.tiny, req), self.tiny) + self.assertEqual(common.limited(self.small, req), self.small) + self.assertEqual(common.limited(self.medium, req), self.medium) + self.assertEqual(common.limited(self.large, req), self.large[:1000]) def test_limiter_limit_medium(self): - """ - Test limit of 10. - """ + """ Test limit of 10. """ req = Request.blank('/?limit=10') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium[:10]) - self.assertEqual(limited(self.large, req), self.large[:10]) + self.assertEqual(common.limited(self.tiny, req), self.tiny) + self.assertEqual(common.limited(self.small, req), self.small) + self.assertEqual(common.limited(self.medium, req), self.medium[:10]) + self.assertEqual(common.limited(self.large, req), self.large[:10]) def test_limiter_limit_over_max(self): - """ - Test limit of 3000. - """ + """ Test limit of 3000. """ req = Request.blank('/?limit=3000') - self.assertEqual(limited(self.tiny, req), self.tiny) - self.assertEqual(limited(self.small, req), self.small) - self.assertEqual(limited(self.medium, req), self.medium) - self.assertEqual(limited(self.large, req), self.large[:1000]) + self.assertEqual(common.limited(self.tiny, req), self.tiny) + self.assertEqual(common.limited(self.small, req), self.small) + self.assertEqual(common.limited(self.medium, req), self.medium) + self.assertEqual(common.limited(self.large, req), self.large[:1000]) def test_limiter_limit_and_offset(self): - """ - Test request with both limit and offset. - """ + """ Test request with both limit and offset. """ items = range(2000) req = Request.blank('/?offset=1&limit=3') - self.assertEqual(limited(items, req), items[1:4]) + self.assertEqual(common.limited(items, req), items[1:4]) req = Request.blank('/?offset=3&limit=0') - self.assertEqual(limited(items, req), items[3:1003]) + self.assertEqual(common.limited(items, req), items[3:1003]) req = Request.blank('/?offset=3&limit=1500') - self.assertEqual(limited(items, req), items[3:1003]) + self.assertEqual(common.limited(items, req), items[3:1003]) req = Request.blank('/?offset=3000&limit=10') - self.assertEqual(limited(items, req), []) + self.assertEqual(common.limited(items, req), []) def test_limiter_custom_max_limit(self): - """ - Test a max_limit other than 1000. - """ + """ Test a max_limit other than 1000. """ items = range(2000) req = Request.blank('/?offset=1&limit=3') - self.assertEqual(limited(items, req, max_limit=2000), items[1:4]) + self.assertEqual( + common.limited(items, req, max_limit=2000), items[1:4]) req = Request.blank('/?offset=3&limit=0') - self.assertEqual(limited(items, req, max_limit=2000), items[3:]) + self.assertEqual( + common.limited(items, req, max_limit=2000), items[3:]) req = Request.blank('/?offset=3&limit=2500') - self.assertEqual(limited(items, req, max_limit=2000), items[3:]) + self.assertEqual( + common.limited(items, req, max_limit=2000), items[3:]) req = Request.blank('/?offset=3000&limit=10') - self.assertEqual(limited(items, req, max_limit=2000), []) + self.assertEqual(common.limited(items, req, max_limit=2000), []) def test_limiter_negative_limit(self): - """ - Test a negative limit. - """ + """ Test a negative limit. """ req = Request.blank('/?limit=-3000') - self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) + self.assertRaises( + webob.exc.HTTPBadRequest, common.limited, self.tiny, req) def test_limiter_negative_offset(self): - """ - Test a negative offset. - """ + """ Test a negative offset. """ req = Request.blank('/?offset=-30') - self.assertRaises(webob.exc.HTTPBadRequest, limited, self.tiny, req) + self.assertRaises( + webob.exc.HTTPBadRequest, common.limited, self.tiny, req) + + +class PaginationParamsTest(test.TestCase): + """ + Unit tests for the `nova.api.openstack.common.get_pagination_params` + method which takes in a request object and returns 'marker' and 'limit' + GET params. + """ + + def test_no_params(self): + """ Test no params. """ + req = Request.blank('/') + self.assertEqual(common.get_pagination_params(req), (0, 0)) + + def test_valid_marker(self): + """ Test valid marker param. """ + req = Request.blank('/?marker=1') + self.assertEqual(common.get_pagination_params(req), (1, 0)) + + def test_invalid_marker(self): + """ Test invalid marker param. """ + req = Request.blank('/?marker=-2') + self.assertRaises( + webob.exc.HTTPBadRequest, common.get_pagination_params, req) + + def test_valid_limit(self): + """ Test valid limit param. """ + req = Request.blank('/?limit=10') + self.assertEqual(common.get_pagination_params(req), (0, 10)) + + def test_invalid_limit(self): + """ Test invalid limit param. """ + req = Request.blank('/?limit=-2') + self.assertRaises( + webob.exc.HTTPBadRequest, common.get_pagination_params, req) diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py index 544298602..697c62e5c 100644 --- a/nova/tests/api/openstack/test_extensions.py +++ b/nova/tests/api/openstack/test_extensions.py @@ -26,15 +26,15 @@ from nova import flags from nova.api import openstack from nova.api.openstack import extensions from nova.api.openstack import flavors +from nova.api.openstack import wsgi from nova.tests.api.openstack import fakes -import nova.wsgi FLAGS = flags.FLAGS response_body = "Try to say this Mr. Knox, sir..." -class StubController(nova.wsgi.Controller): +class StubController(object): def __init__(self, body): self.body = body @@ -128,6 +128,11 @@ class ResourceExtensionTest(unittest.TestCase): self.assertEqual(response_body, response.body) +class InvalidExtension(object): + def get_alias(self): + return "THIRD" + + class ExtensionManagerTest(unittest.TestCase): response_body = "Try to say this Mr. Knox, sir..." @@ -144,6 +149,14 @@ class ExtensionManagerTest(unittest.TestCase): self.assertEqual(200, response.status_int) self.assertEqual(response_body, response.body) + def test_invalid_extensions(self): + app = openstack.APIRouterV11() + ext_midware = extensions.ExtensionMiddleware(app) + ext_mgr = ext_midware.ext_mgr + ext_mgr.add_extension(InvalidExtension()) + self.assertTrue('FOXNSOX' in ext_mgr.extensions) + self.assertTrue('THIRD' not in ext_mgr.extensions) + class ActionExtensionTest(unittest.TestCase): diff --git a/nova/tests/api/openstack/test_image_metadata.py b/nova/tests/api/openstack/test_image_metadata.py index 56be0f1cc..730af3665 100644 --- a/nova/tests/api/openstack/test_image_metadata.py +++ b/nova/tests/api/openstack/test_image_metadata.py @@ -19,6 +19,7 @@ import json import stubout import unittest import webob +import xml.dom.minidom as minidom from nova import flags @@ -37,6 +38,7 @@ class ImageMetaDataTest(unittest.TestCase): 'name': 'image1', 'deleted': False, 'container_format': None, + 'checksum': None, 'created_at': '2011-03-22T17:40:15', 'disk_format': None, 'updated_at': '2011-03-22T17:40:15', @@ -52,6 +54,7 @@ class ImageMetaDataTest(unittest.TestCase): 'name': 'image2', 'deleted': False, 'container_format': None, + 'checksum': None, 'created_at': '2011-03-22T17:40:15', 'disk_format': None, 'updated_at': '2011-03-22T17:40:15', @@ -67,6 +70,7 @@ class ImageMetaDataTest(unittest.TestCase): 'name': 'image3', 'deleted': False, 'container_format': None, + 'checksum': None, 'created_at': '2011-03-22T17:40:15', 'disk_format': None, 'updated_at': '2011-03-22T17:40:15', @@ -103,7 +107,34 @@ class ImageMetaDataTest(unittest.TestCase): res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(200, res.status_int) - self.assertEqual('value1', res_dict['metadata']['key1']) + expected = self.IMAGE_FIXTURES[0]['properties'] + self.assertEqual(len(expected), len(res_dict['metadata'])) + for (key, value) in res_dict['metadata'].items(): + self.assertEqual(value, res_dict['metadata'][key]) + + def test_index_xml(self): + serializer = openstack.image_metadata.ImageMetadataXMLSerializer() + fixture = { + 'metadata': { + 'one': 'two', + 'three': 'four', + }, + } + output = serializer.index(fixture) + actual = minidom.parseString(output.replace(" ", "")) + + expected = minidom.parseString(""" + <metadata xmlns="http://docs.openstack.org/compute/api/v1.1"> + <meta key="three"> + four + </meta> + <meta key="one"> + two + </meta> + </metadata> + """.replace(" ", "")) + + self.assertEqual(expected.toxml(), actual.toxml()) def test_show(self): req = webob.Request.blank('/v1.1/images/1/meta/key1') @@ -111,13 +142,32 @@ class ImageMetaDataTest(unittest.TestCase): res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(200, res.status_int) - self.assertEqual('value1', res_dict['key1']) + self.assertTrue('meta' in res_dict) + self.assertEqual(len(res_dict['meta']), 1) + self.assertEqual('value1', res_dict['meta']['key1']) + + def test_show_xml(self): + serializer = openstack.image_metadata.ImageMetadataXMLSerializer() + fixture = { + 'meta': { + 'one': 'two', + }, + } + output = serializer.show(fixture) + actual = minidom.parseString(output.replace(" ", "")) + + expected = minidom.parseString(""" + <meta xmlns="http://docs.openstack.org/compute/api/v1.1" key="one"> + two + </meta> + """.replace(" ", "")) + + self.assertEqual(expected.toxml(), actual.toxml()) def test_show_not_found(self): req = webob.Request.blank('/v1.1/images/1/meta/key9') req.environ['api.version'] = '1.1' res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(404, res.status_int) def test_create(self): @@ -135,22 +185,79 @@ class ImageMetaDataTest(unittest.TestCase): self.assertEqual('value2', res_dict['metadata']['key2']) self.assertEqual(1, len(res_dict)) + def test_create_xml(self): + serializer = openstack.image_metadata.ImageMetadataXMLSerializer() + fixture = { + 'metadata': { + 'key9': 'value9', + 'key2': 'value2', + 'key1': 'value1', + }, + } + output = serializer.create(fixture) + actual = minidom.parseString(output.replace(" ", "")) + + expected = minidom.parseString(""" + <metadata xmlns="http://docs.openstack.org/compute/api/v1.1"> + <meta key="key2"> + value2 + </meta> + <meta key="key9"> + value9 + </meta> + <meta key="key1"> + value1 + </meta> + </metadata> + """.replace(" ", "")) + + self.assertEqual(expected.toxml(), actual.toxml()) + def test_update_item(self): req = webob.Request.blank('/v1.1/images/1/meta/key1') req.environ['api.version'] = '1.1' req.method = 'PUT' - req.body = '{"key1": "zz"}' + req.body = '{"meta": {"key1": "zz"}}' req.headers["content-type"] = "application/json" res = req.get_response(fakes.wsgi_app()) self.assertEqual(200, res.status_int) res_dict = json.loads(res.body) - self.assertEqual('zz', res_dict['key1']) + self.assertTrue('meta' in res_dict) + self.assertEqual(len(res_dict['meta']), 1) + self.assertEqual('zz', res_dict['meta']['key1']) + + def test_update_item_bad_body(self): + req = webob.Request.blank('/v1.1/images/1/meta/key1') + req.environ['api.version'] = '1.1' + req.method = 'PUT' + req.body = '{"key1": "zz"}' + req.headers["content-type"] = "application/json" + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(400, res.status_int) + + def test_update_item_xml(self): + serializer = openstack.image_metadata.ImageMetadataXMLSerializer() + fixture = { + 'meta': { + 'one': 'two', + }, + } + output = serializer.update(fixture) + actual = minidom.parseString(output.replace(" ", "")) + + expected = minidom.parseString(""" + <meta xmlns="http://docs.openstack.org/compute/api/v1.1" key="one"> + two + </meta> + """.replace(" ", "")) + + self.assertEqual(expected.toxml(), actual.toxml()) def test_update_item_too_many_keys(self): req = webob.Request.blank('/v1.1/images/1/meta/key1') req.environ['api.version'] = '1.1' req.method = 'PUT' - req.body = '{"key1": "value1", "key2": "value2"}' + req.body = '{"meta": {"key1": "value1", "key2": "value2"}}' req.headers["content-type"] = "application/json" res = req.get_response(fakes.wsgi_app()) self.assertEqual(400, res.status_int) @@ -159,7 +266,7 @@ class ImageMetaDataTest(unittest.TestCase): req = webob.Request.blank('/v1.1/images/1/meta/bad') req.environ['api.version'] = '1.1' req.method = 'PUT' - req.body = '{"key1": "value1"}' + req.body = '{"meta": {"key1": "value1"}}' req.headers["content-type"] = "application/json" res = req.get_response(fakes.wsgi_app()) self.assertEqual(400, res.status_int) @@ -195,7 +302,7 @@ class ImageMetaDataTest(unittest.TestCase): req = webob.Request.blank('/v1.1/images/3/meta/blah') req.environ['api.version'] = '1.1' req.method = 'PUT' - req.body = '{"blah": "blah"}' + req.body = '{"meta": {"blah": "blah"}}' req.headers["content-type"] = "application/json" res = req.get_response(fakes.wsgi_app()) self.assertEqual(400, res.status_int) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 2c329f920..446d68e9e 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -22,12 +22,12 @@ and as a WSGI layer import copy import json -import datetime import os import shutil import tempfile import xml.dom.minidom as minidom +import mox import stubout import webob @@ -127,7 +127,7 @@ class _BaseImageServiceTests(test.TestCase): @staticmethod def _make_fixture(name): - fixture = {'name': 'test image', + fixture = {'name': name, 'updated': None, 'created': None, 'status': None, @@ -135,36 +135,6 @@ class _BaseImageServiceTests(test.TestCase): return fixture -class LocalImageServiceTest(_BaseImageServiceTests): - - """Tests the local image service""" - - def setUp(self): - super(LocalImageServiceTest, self).setUp() - self.tempdir = tempfile.mkdtemp() - self.flags(images_path=self.tempdir) - self.stubs = stubout.StubOutForTesting() - service_class = 'nova.image.local.LocalImageService' - self.service = utils.import_object(service_class) - self.context = context.RequestContext(None, None) - - def tearDown(self): - shutil.rmtree(self.tempdir) - self.stubs.UnsetAll() - super(LocalImageServiceTest, self).tearDown() - - def test_get_all_ids_with_incorrect_directory_formats(self): - # create some old-style image directories (starting with 'ami-') - for x in [1, 2, 3]: - tempfile.mkstemp(prefix='ami-', dir=self.tempdir) - # create some valid image directories names - for x in ["1485baed", "1a60f0ee", "3123a73d"]: - os.makedirs(os.path.join(self.tempdir, x)) - found_image_ids = self.service._ids() - self.assertEqual(True, isinstance(found_image_ids, list)) - self.assertEqual(3, len(found_image_ids), len(found_image_ids)) - - class GlanceImageServiceTest(_BaseImageServiceTests): """Tests the Glance image service, in particular that metadata translation @@ -226,6 +196,127 @@ class GlanceImageServiceTest(_BaseImageServiceTests): expected = {'name': 'test image', 'properties': {}} self.assertDictMatch(self.sent_to_glance['metadata'], expected) + def test_index_default_limit(self): + fixtures = [] + ids = [] + for i in range(10): + fixture = self._make_fixture('TestImage %d' % (i)) + fixtures.append(fixture) + ids.append(self.service.create(self.context, fixture)['id']) + + image_metas = self.service.index(self.context) + i = 0 + for meta in image_metas: + expected = {'id': 'DONTCARE', + 'name': 'TestImage %d' % (i)} + self.assertDictMatch(meta, expected) + i = i + 1 + + def test_index_marker(self): + fixtures = [] + ids = [] + for i in range(10): + fixture = self._make_fixture('TestImage %d' % (i)) + fixtures.append(fixture) + ids.append(self.service.create(self.context, fixture)['id']) + + image_metas = self.service.index(self.context, marker=ids[1]) + self.assertEquals(len(image_metas), 8) + i = 2 + for meta in image_metas: + expected = {'id': 'DONTCARE', + 'name': 'TestImage %d' % (i)} + self.assertDictMatch(meta, expected) + i = i + 1 + + def test_index_limit(self): + fixtures = [] + ids = [] + for i in range(10): + fixture = self._make_fixture('TestImage %d' % (i)) + fixtures.append(fixture) + ids.append(self.service.create(self.context, fixture)['id']) + + image_metas = self.service.index(self.context, limit=3) + self.assertEquals(len(image_metas), 3) + + def test_index_marker_and_limit(self): + fixtures = [] + ids = [] + for i in range(10): + fixture = self._make_fixture('TestImage %d' % (i)) + fixtures.append(fixture) + ids.append(self.service.create(self.context, fixture)['id']) + + image_metas = self.service.index(self.context, marker=ids[3], limit=1) + self.assertEquals(len(image_metas), 1) + i = 4 + for meta in image_metas: + expected = {'id': 'DONTCARE', + 'name': 'TestImage %d' % (i)} + self.assertDictMatch(meta, expected) + i = i + 1 + + def test_detail_marker(self): + fixtures = [] + ids = [] + for i in range(10): + fixture = self._make_fixture('TestImage %d' % (i)) + fixtures.append(fixture) + ids.append(self.service.create(self.context, fixture)['id']) + + image_metas = self.service.detail(self.context, marker=ids[1]) + self.assertEquals(len(image_metas), 8) + i = 2 + for meta in image_metas: + expected = { + 'id': 'DONTCARE', + 'status': None, + 'is_public': True, + 'name': 'TestImage %d' % (i), + 'properties': { + 'updated': None, + 'created': None, + }, + } + + self.assertDictMatch(meta, expected) + i = i + 1 + + def test_detail_limit(self): + fixtures = [] + ids = [] + for i in range(10): + fixture = self._make_fixture('TestImage %d' % (i)) + fixtures.append(fixture) + ids.append(self.service.create(self.context, fixture)['id']) + + image_metas = self.service.detail(self.context, limit=3) + self.assertEquals(len(image_metas), 3) + + def test_detail_marker_and_limit(self): + fixtures = [] + ids = [] + for i in range(10): + fixture = self._make_fixture('TestImage %d' % (i)) + fixtures.append(fixture) + ids.append(self.service.create(self.context, fixture)['id']) + + image_metas = self.service.detail(self.context, marker=ids[3], limit=3) + self.assertEquals(len(image_metas), 3) + i = 4 + for meta in image_metas: + expected = { + 'id': 'DONTCARE', + 'status': None, + 'is_public': True, + 'name': 'TestImage %d' % (i), + 'properties': { + 'updated': None, 'created': None}, + } + self.assertDictMatch(meta, expected) + i = i + 1 + class ImageControllerWithGlanceServiceTest(test.TestCase): """ @@ -248,6 +339,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): fakes.stub_out_key_pair_funcs(self.stubs) self.fixtures = self._make_image_fixtures() fakes.stub_out_glance(self.stubs, initial_fixtures=self.fixtures) + fakes.stub_out_compute_api_snapshot(self.stubs) def tearDown(self): """Run after each test.""" @@ -526,7 +618,6 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): { 'id': 124, 'name': 'queued backup', - 'serverId': 42, 'updated': self.NOW_API_FORMAT, 'created': self.NOW_API_FORMAT, 'status': 'QUEUED', @@ -534,7 +625,6 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): { 'id': 125, 'name': 'saving backup', - 'serverId': 42, 'updated': self.NOW_API_FORMAT, 'created': self.NOW_API_FORMAT, 'status': 'SAVING', @@ -543,7 +633,6 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): { 'id': 126, 'name': 'active backup', - 'serverId': 42, 'updated': self.NOW_API_FORMAT, 'created': self.NOW_API_FORMAT, 'status': 'ACTIVE' @@ -551,7 +640,6 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): { 'id': 127, 'name': 'killed backup', - 'serverId': 42, 'updated': self.NOW_API_FORMAT, 'created': self.NOW_API_FORMAT, 'status': 'FAILED', @@ -597,7 +685,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): { 'id': 124, 'name': 'queued backup', - 'serverRef': "http://localhost/v1.1/servers/42", + 'serverRef': "http://localhost:8774/v1.1/servers/42", 'updated': self.NOW_API_FORMAT, 'created': self.NOW_API_FORMAT, 'status': 'QUEUED', @@ -619,7 +707,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): { 'id': 125, 'name': 'saving backup', - 'serverRef': "http://localhost/v1.1/servers/42", + 'serverRef': "http://localhost:8774/v1.1/servers/42", 'updated': self.NOW_API_FORMAT, 'created': self.NOW_API_FORMAT, 'status': 'SAVING', @@ -642,7 +730,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): { 'id': 126, 'name': 'active backup', - 'serverRef': "http://localhost/v1.1/servers/42", + 'serverRef': "http://localhost:8774/v1.1/servers/42", 'updated': self.NOW_API_FORMAT, 'created': self.NOW_API_FORMAT, 'status': 'ACTIVE', @@ -664,7 +752,7 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): { 'id': 127, 'name': 'killed backup', - 'serverRef': "http://localhost/v1.1/servers/42", + 'serverRef': "http://localhost:8774/v1.1/servers/42", 'updated': self.NOW_API_FORMAT, 'created': self.NOW_API_FORMAT, 'status': 'FAILED', @@ -708,6 +796,156 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): self.assertDictListMatch(expected, response_list) + def test_image_filter_with_name(self): + mocker = mox.Mox() + image_service = mocker.CreateMockAnything() + context = object() + filters = {'name': 'testname'} + image_service.index( + context, filters=filters, marker=0, limit=0).AndReturn([]) + mocker.ReplayAll() + request = webob.Request.blank( + '/v1.1/images?name=testname') + request.environ['nova.context'] = context + controller = images.ControllerV11(image_service=image_service) + controller.index(request) + mocker.VerifyAll() + + def test_image_filter_with_status(self): + mocker = mox.Mox() + image_service = mocker.CreateMockAnything() + context = object() + filters = {'status': 'ACTIVE'} + image_service.index( + context, filters=filters, marker=0, limit=0).AndReturn([]) + mocker.ReplayAll() + request = webob.Request.blank( + '/v1.1/images?status=ACTIVE') + request.environ['nova.context'] = context + controller = images.ControllerV11(image_service=image_service) + controller.index(request) + mocker.VerifyAll() + + def test_image_filter_with_property(self): + mocker = mox.Mox() + image_service = mocker.CreateMockAnything() + context = object() + filters = {'property-test': '3'} + image_service.index( + context, filters=filters, marker=0, limit=0).AndReturn([]) + mocker.ReplayAll() + request = webob.Request.blank( + '/v1.1/images?property-test=3') + request.environ['nova.context'] = context + controller = images.ControllerV11(image_service=image_service) + controller.index(request) + mocker.VerifyAll() + + def test_image_filter_not_supported(self): + mocker = mox.Mox() + image_service = mocker.CreateMockAnything() + context = object() + filters = {'status': 'ACTIVE'} + image_service.index( + context, filters=filters, marker=0, limit=0).AndReturn([]) + mocker.ReplayAll() + request = webob.Request.blank( + '/v1.1/images?status=ACTIVE&UNSUPPORTEDFILTER=testname') + request.environ['nova.context'] = context + controller = images.ControllerV11(image_service=image_service) + controller.index(request) + mocker.VerifyAll() + + def test_image_no_filters(self): + mocker = mox.Mox() + image_service = mocker.CreateMockAnything() + context = object() + filters = {} + image_service.index( + context, filters=filters, marker=0, limit=0).AndReturn([]) + mocker.ReplayAll() + request = webob.Request.blank( + '/v1.1/images') + request.environ['nova.context'] = context + controller = images.ControllerV11(image_service=image_service) + controller.index(request) + mocker.VerifyAll() + + def test_image_detail_filter_with_name(self): + mocker = mox.Mox() + image_service = mocker.CreateMockAnything() + context = object() + filters = {'name': 'testname'} + image_service.detail( + context, filters=filters, marker=0, limit=0).AndReturn([]) + mocker.ReplayAll() + request = webob.Request.blank( + '/v1.1/images/detail?name=testname') + request.environ['nova.context'] = context + controller = images.ControllerV11(image_service=image_service) + controller.detail(request) + mocker.VerifyAll() + + def test_image_detail_filter_with_status(self): + mocker = mox.Mox() + image_service = mocker.CreateMockAnything() + context = object() + filters = {'status': 'ACTIVE'} + image_service.detail( + context, filters=filters, marker=0, limit=0).AndReturn([]) + mocker.ReplayAll() + request = webob.Request.blank( + '/v1.1/images/detail?status=ACTIVE') + request.environ['nova.context'] = context + controller = images.ControllerV11(image_service=image_service) + controller.detail(request) + mocker.VerifyAll() + + def test_image_detail_filter_with_property(self): + mocker = mox.Mox() + image_service = mocker.CreateMockAnything() + context = object() + filters = {'property-test': '3'} + image_service.detail( + context, filters=filters, marker=0, limit=0).AndReturn([]) + mocker.ReplayAll() + request = webob.Request.blank( + '/v1.1/images/detail?property-test=3') + request.environ['nova.context'] = context + controller = images.ControllerV11(image_service=image_service) + controller.detail(request) + mocker.VerifyAll() + + def test_image_detail_filter_not_supported(self): + mocker = mox.Mox() + image_service = mocker.CreateMockAnything() + context = object() + filters = {'status': 'ACTIVE'} + image_service.detail( + context, filters=filters, marker=0, limit=0).AndReturn([]) + mocker.ReplayAll() + request = webob.Request.blank( + '/v1.1/images/detail?status=ACTIVE&UNSUPPORTEDFILTER=testname') + request.environ['nova.context'] = context + controller = images.ControllerV11(image_service=image_service) + controller.detail(request) + mocker.VerifyAll() + + def test_image_detail_no_filters(self): + mocker = mox.Mox() + image_service = mocker.CreateMockAnything() + context = object() + filters = {} + image_service.detail( + context, filters=filters, marker=0, limit=0).AndReturn([]) + mocker.ReplayAll() + request = webob.Request.blank( + '/v1.1/images/detail') + request.environ['nova.context'] = context + controller = images.ControllerV11(image_service=image_service) + controller.detail(request) + mocker.VerifyAll() + def test_get_image_found(self): req = webob.Request.blank('/v1.0/images/123') res = req.get_response(fakes.wsgi_app()) @@ -730,6 +968,103 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 404) + def test_create_image(self): + + body = dict(image=dict(serverId='123', name='Backup 1')) + req = webob.Request.blank('/v1.0/images') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + response = req.get_response(fakes.wsgi_app()) + self.assertEqual(200, response.status_int) + + def test_create_image_no_server_id(self): + + body = dict(image=dict(name='Backup 1')) + req = webob.Request.blank('/v1.0/images') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + response = req.get_response(fakes.wsgi_app()) + self.assertEqual(400, response.status_int) + + def test_create_image_v1_1(self): + + body = dict(image=dict(serverRef='123', name='Backup 1')) + req = webob.Request.blank('/v1.1/images') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + response = req.get_response(fakes.wsgi_app()) + self.assertEqual(200, response.status_int) + + def test_create_image_v1_1_actual_server_ref(self): + + serverRef = 'http://localhost/v1.1/servers/1' + body = dict(image=dict(serverRef=serverRef, name='Backup 1')) + req = webob.Request.blank('/v1.1/images') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + response = req.get_response(fakes.wsgi_app()) + self.assertEqual(200, response.status_int) + result = json.loads(response.body) + self.assertEqual(result['image']['serverRef'], serverRef) + + def test_create_image_v1_1_server_ref_bad_hostname(self): + + serverRef = 'http://asdf/v1.1/servers/1' + body = dict(image=dict(serverRef=serverRef, name='Backup 1')) + req = webob.Request.blank('/v1.1/images') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + response = req.get_response(fakes.wsgi_app()) + self.assertEqual(400, response.status_int) + + def test_create_image_v1_1_xml_serialization(self): + + body = dict(image=dict(serverRef='123', name='Backup 1')) + req = webob.Request.blank('/v1.1/images') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + req.headers["accept"] = "application/xml" + response = req.get_response(fakes.wsgi_app()) + self.assertEqual(200, response.status_int) + resp_xml = minidom.parseString(response.body.replace(" ", "")) + expected_href = "http://localhost/v1.1/images/123" + expected_image = minidom.parseString(""" + <image + created="None" + id="123" + name="Backup 1" + serverRef="http://localhost/v1.1/servers/123" + status="ACTIVE" + updated="None" + xmlns="http://docs.openstack.org/compute/api/v1.1"> + <links> + <link href="%(expected_href)s" rel="self"/> + <link href="%(expected_href)s" rel="bookmark" + type="application/json" /> + <link href="%(expected_href)s" rel="bookmark" + type="application/xml" /> + </links> + </image> + """.replace(" ", "") % (locals())) + + self.assertEqual(expected_image.toxml(), resp_xml.toxml()) + + def test_create_image_v1_1_no_server_ref(self): + + body = dict(image=dict(name='Backup 1')) + req = webob.Request.blank('/v1.1/images') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + response = req.get_response(fakes.wsgi_app()) + self.assertEqual(400, response.status_int) + @classmethod def _make_image_fixtures(cls): image_id = 123 @@ -750,7 +1085,8 @@ class ImageControllerWithGlanceServiceTest(test.TestCase): image_id += 1 # Backup for User 1 - backup_properties = {'instance_id': '42', 'user_id': '1'} + server_ref = 'http://localhost:8774/v1.1/servers/42' + backup_properties = {'instance_ref': server_ref, 'user_id': '1'} for status in ('queued', 'saving', 'active', 'killed'): add_fixture(id=image_id, name='%s backup' % status, is_public=False, status=status, diff --git a/nova/tests/api/openstack/test_limits.py b/nova/tests/api/openstack/test_limits.py index 45bd4d501..38c959fae 100644 --- a/nova/tests/api/openstack/test_limits.py +++ b/nova/tests/api/openstack/test_limits.py @@ -27,6 +27,7 @@ import webob from xml.dom.minidom import parseString +import nova.context from nova.api.openstack import limits @@ -47,6 +48,13 @@ class BaseLimitTestSuite(unittest.TestCase): self.time = 0.0 self.stubs = stubout.StubOutForTesting() self.stubs.Set(limits.Limit, "_get_time", self._get_time) + self.absolute_limits = {} + + def stub_get_project_quotas(context, project_id): + return self.absolute_limits + + self.stubs.Set(nova.quota, "get_project_quotas", + stub_get_project_quotas) def tearDown(self): """Run after each test.""" @@ -65,7 +73,7 @@ class LimitsControllerV10Test(BaseLimitTestSuite): def setUp(self): """Run before each test.""" BaseLimitTestSuite.setUp(self) - self.controller = limits.LimitsControllerV10() + self.controller = limits.create_resource('1.0') def _get_index_request(self, accept_header="application/json"): """Helper to set routing arguments.""" @@ -75,6 +83,8 @@ class LimitsControllerV10Test(BaseLimitTestSuite): "action": "index", "controller": "", }) + context = nova.context.RequestContext('testuser', 'testproject') + request.environ["nova.context"] = context return request def _populate_limits(self, request): @@ -86,6 +96,18 @@ class LimitsControllerV10Test(BaseLimitTestSuite): request.environ["nova.limits"] = _limits return request + def _setup_absolute_limits(self): + self.absolute_limits = { + 'instances': 5, + 'cores': 8, + 'ram': 2 ** 13, + 'volumes': 21, + 'gigabytes': 34, + 'metadata_items': 55, + 'injected_files': 89, + 'injected_file_content_bytes': 144, + } + def test_empty_index_json(self): """Test getting empty limit details in JSON.""" request = self._get_index_request() @@ -103,6 +125,7 @@ class LimitsControllerV10Test(BaseLimitTestSuite): """Test getting limit details in JSON.""" request = self._get_index_request() request = self._populate_limits(request) + self._setup_absolute_limits() response = request.get_response(self.controller) expected = { "limits": { @@ -124,7 +147,15 @@ class LimitsControllerV10Test(BaseLimitTestSuite): "remaining": 5, "unit": "HOUR", }], - "absolute": {}, + "absolute": { + "maxTotalInstances": 5, + "maxTotalCores": 8, + "maxTotalRAMSize": 2 ** 13, + "maxServerMeta": 55, + "maxImageMeta": 55, + "maxPersonality": 89, + "maxPersonalitySize": 144, + }, }, } body = json.loads(response.body) @@ -178,7 +209,7 @@ class LimitsControllerV11Test(BaseLimitTestSuite): def setUp(self): """Run before each test.""" BaseLimitTestSuite.setUp(self) - self.controller = limits.LimitsControllerV11() + self.controller = limits.create_resource('1.1') def _get_index_request(self, accept_header="application/json"): """Helper to set routing arguments.""" @@ -188,6 +219,8 @@ class LimitsControllerV11Test(BaseLimitTestSuite): "action": "index", "controller": "", }) + context = nova.context.RequestContext('testuser', 'testproject') + request.environ["nova.context"] = context return request def _populate_limits(self, request): @@ -218,6 +251,11 @@ class LimitsControllerV11Test(BaseLimitTestSuite): """Test getting limit details in JSON.""" request = self._get_index_request() request = self._populate_limits(request) + self.absolute_limits = { + 'ram': 512, + 'instances': 5, + 'cores': 21, + } response = request.get_response(self.controller) expected = { "limits": { @@ -257,12 +295,110 @@ class LimitsControllerV11Test(BaseLimitTestSuite): }, ], + "absolute": { + "maxTotalRAMSize": 512, + "maxTotalInstances": 5, + "maxTotalCores": 21, + }, + }, + } + body = json.loads(response.body) + self.assertEqual(expected, body) + + def _populate_limits_diff_regex(self, request): + """Put limit info into a request.""" + _limits = [ + limits.Limit("GET", "*", ".*", 10, 60).display(), + limits.Limit("GET", "*", "*.*", 10, 60).display(), + ] + request.environ["nova.limits"] = _limits + return request + + def test_index_diff_regex(self): + """Test getting limit details in JSON.""" + request = self._get_index_request() + request = self._populate_limits_diff_regex(request) + response = request.get_response(self.controller) + expected = { + "limits": { + "rate": [ + { + "regex": ".*", + "uri": "*", + "limit": [ + { + "verb": "GET", + "next-available": 0, + "unit": "MINUTE", + "value": 10, + "remaining": 10, + }, + ], + }, + { + "regex": "*.*", + "uri": "*", + "limit": [ + { + "verb": "GET", + "next-available": 0, + "unit": "MINUTE", + "value": 10, + "remaining": 10, + }, + ], + }, + + ], "absolute": {}, }, } body = json.loads(response.body) self.assertEqual(expected, body) + def _test_index_absolute_limits_json(self, expected): + request = self._get_index_request() + response = request.get_response(self.controller) + body = json.loads(response.body) + self.assertEqual(expected, body['limits']['absolute']) + + def test_index_ignores_extra_absolute_limits_json(self): + self.absolute_limits = {'unknown_limit': 9001} + self._test_index_absolute_limits_json({}) + + def test_index_absolute_ram_json(self): + self.absolute_limits = {'ram': 1024} + self._test_index_absolute_limits_json({'maxTotalRAMSize': 1024}) + + def test_index_absolute_cores_json(self): + self.absolute_limits = {'cores': 17} + self._test_index_absolute_limits_json({'maxTotalCores': 17}) + + def test_index_absolute_instances_json(self): + self.absolute_limits = {'instances': 19} + self._test_index_absolute_limits_json({'maxTotalInstances': 19}) + + def test_index_absolute_metadata_json(self): + # NOTE: both server metadata and image metadata are overloaded + # into metadata_items + self.absolute_limits = {'metadata_items': 23} + expected = { + 'maxServerMeta': 23, + 'maxImageMeta': 23, + } + self._test_index_absolute_limits_json(expected) + + def test_index_absolute_injected_files(self): + self.absolute_limits = { + 'injected_files': 17, + 'injected_file_content_bytes': 86753, + } + expected = { + 'maxPersonality': 17, + 'maxPersonalitySize': 86753, + } + self._test_index_absolute_limits_json(expected) + class LimitMiddlewareTest(BaseLimitTestSuite): """ @@ -536,8 +672,7 @@ class WsgiLimiterTest(BaseLimitTestSuite): """Only POSTs should work.""" requests = [] for method in ["GET", "PUT", "DELETE", "HEAD", "OPTIONS"]: - request = webob.Request.blank("/") - request.body = self._request_data("GET", "/something") + request = webob.Request.blank("/", method=method) response = request.get_response(self.app) self.assertEqual(response.status_int, 405) diff --git a/nova/tests/api/openstack/test_server_metadata.py b/nova/tests/api/openstack/test_server_metadata.py index c4d1d4fd8..0431e68d2 100644 --- a/nova/tests/api/openstack/test_server_metadata.py +++ b/nova/tests/api/openstack/test_server_metadata.py @@ -21,6 +21,7 @@ import unittest import webob +from nova import exception from nova import flags from nova.api import openstack from nova.tests.api.openstack import fakes @@ -67,6 +68,14 @@ def stub_max_server_metadata(): return metadata +def return_server(context, server_id): + return {'id': server_id} + + +def return_server_nonexistant(context, server_id): + raise exception.InstanceNotFound() + + class ServerMetaDataTest(unittest.TestCase): def setUp(self): @@ -76,6 +85,7 @@ class ServerMetaDataTest(unittest.TestCase): fakes.FakeAuthDatabase.data = {} fakes.stub_out_auth(self.stubs) fakes.stub_out_key_pair_funcs(self.stubs) + self.stubs.Set(nova.db.api, 'instance_get', return_server) def tearDown(self): self.stubs.UnsetAll() @@ -89,8 +99,16 @@ class ServerMetaDataTest(unittest.TestCase): res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(200, res.status_int) + self.assertEqual('application/json', res.headers['Content-Type']) self.assertEqual('value1', res_dict['metadata']['key1']) + def test_index_nonexistant_server(self): + self.stubs.Set(nova.db.api, 'instance_get', return_server_nonexistant) + req = webob.Request.blank('/v1.1/servers/1/meta') + req.environ['api.version'] = '1.1' + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(404, res.status_int) + def test_index_no_data(self): self.stubs.Set(nova.db.api, 'instance_metadata_get', return_empty_server_metadata) @@ -99,6 +117,7 @@ class ServerMetaDataTest(unittest.TestCase): res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(200, res.status_int) + self.assertEqual('application/json', res.headers['Content-Type']) self.assertEqual(0, len(res_dict['metadata'])) def test_show(self): @@ -109,15 +128,22 @@ class ServerMetaDataTest(unittest.TestCase): res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(200, res.status_int) + self.assertEqual('application/json', res.headers['Content-Type']) self.assertEqual('value5', res_dict['key5']) + def test_show_nonexistant_server(self): + self.stubs.Set(nova.db.api, 'instance_get', return_server_nonexistant) + req = webob.Request.blank('/v1.1/servers/1/meta/key5') + req.environ['api.version'] = '1.1' + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(404, res.status_int) + def test_show_meta_not_found(self): self.stubs.Set(nova.db.api, 'instance_metadata_get', return_empty_server_metadata) req = webob.Request.blank('/v1.1/servers/1/meta/key6') req.environ['api.version'] = '1.1' res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(404, res.status_int) def test_delete(self): @@ -129,6 +155,14 @@ class ServerMetaDataTest(unittest.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(200, res.status_int) + def test_delete_nonexistant_server(self): + self.stubs.Set(nova.db.api, 'instance_get', return_server_nonexistant) + req = webob.Request.blank('/v1.1/servers/1/meta/key5') + req.environ['api.version'] = '1.1' + req.method = 'DELETE' + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(404, res.status_int) + def test_create(self): self.stubs.Set(nova.db.api, 'instance_metadata_update_or_create', return_create_instance_metadata) @@ -138,10 +172,31 @@ class ServerMetaDataTest(unittest.TestCase): req.body = '{"metadata": {"key1": "value1"}}' req.headers["content-type"] = "application/json" res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(200, res.status_int) + res_dict = json.loads(res.body) + self.assertEqual('application/json', res.headers['Content-Type']) self.assertEqual('value1', res_dict['metadata']['key1']) + def test_create_empty_body(self): + self.stubs.Set(nova.db.api, 'instance_metadata_update_or_create', + return_create_instance_metadata) + req = webob.Request.blank('/v1.1/servers/1/meta') + req.environ['api.version'] = '1.1' + req.method = 'POST' + req.headers["content-type"] = "application/json" + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(400, res.status_int) + + def test_create_nonexistant_server(self): + self.stubs.Set(nova.db.api, 'instance_get', return_server_nonexistant) + req = webob.Request.blank('/v1.1/servers/100/meta') + req.environ['api.version'] = '1.1' + req.method = 'POST' + req.body = '{"metadata": {"key1": "value1"}}' + req.headers["content-type"] = "application/json" + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(404, res.status_int) + def test_update_item(self): self.stubs.Set(nova.db.api, 'instance_metadata_update_or_create', return_create_instance_metadata) @@ -152,9 +207,30 @@ class ServerMetaDataTest(unittest.TestCase): req.headers["content-type"] = "application/json" res = req.get_response(fakes.wsgi_app()) self.assertEqual(200, res.status_int) + self.assertEqual('application/json', res.headers['Content-Type']) res_dict = json.loads(res.body) self.assertEqual('value1', res_dict['key1']) + def test_update_item_nonexistant_server(self): + self.stubs.Set(nova.db.api, 'instance_get', return_server_nonexistant) + req = webob.Request.blank('/v1.1/servers/asdf/100/key1') + req.environ['api.version'] = '1.1' + req.method = 'PUT' + req.body = '{"key1": "value1"}' + req.headers["content-type"] = "application/json" + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(404, res.status_int) + + def test_update_item_empty_body(self): + self.stubs.Set(nova.db.api, 'instance_metadata_update_or_create', + return_create_instance_metadata) + req = webob.Request.blank('/v1.1/servers/1/meta/key1') + req.environ['api.version'] = '1.1' + req.method = 'PUT' + req.headers["content-type"] = "application/json" + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(400, res.status_int) + def test_update_item_too_many_keys(self): self.stubs.Set(nova.db.api, 'instance_metadata_update_or_create', return_create_instance_metadata) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index e8182b6a9..b53c6c9be 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -16,7 +16,6 @@ # under the License. import base64 -import datetime import json import unittest from xml.dom import minidom @@ -29,14 +28,18 @@ from nova import db from nova import exception from nova import flags from nova import test +from nova import utils import nova.api.openstack from nova.api.openstack import servers +from nova.api.openstack import create_instance_helper import nova.compute.api from nova.compute import instance_types from nova.compute import power_state import nova.db.api +import nova.scheduler.api from nova.db.sqlalchemy.models import Instance from nova.db.sqlalchemy.models import InstanceMetadata +import nova.image.fake import nova.rpc from nova.tests.api.openstack import common from nova.tests.api.openstack import fakes @@ -46,10 +49,22 @@ FLAGS = flags.FLAGS FLAGS.verbose = True -def return_server(context, id): +FAKE_UUID = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' + + +def fake_gen_uuid(): + return FAKE_UUID + + +def return_server_by_id(context, id): return stub_instance(id) +def return_server_by_uuid(context, uuid): + id = 1 + return stub_instance(id, uuid=uuid) + + def return_server_with_addresses(private, public): def _return_server(context, id): return stub_instance(id, private_address=private, @@ -67,6 +82,34 @@ def return_servers(context, user_id=1): return [stub_instance(i, user_id) for i in xrange(5)] +def return_servers_by_reservation(context, reservation_id=""): + return [stub_instance(i, reservation_id) for i in xrange(5)] + + +def return_servers_by_reservation_empty(context, reservation_id=""): + return [] + + +def return_servers_from_child_zones_empty(*args, **kwargs): + return [] + + +def return_servers_from_child_zones(*args, **kwargs): + class Server(object): + pass + + zones = [] + for zone in xrange(3): + servers = [] + for server_id in xrange(5): + server = Server() + server._info = stub_instance(server_id, reservation_id="child") + servers.append(server) + + zones.append(("Zone%d" % zone, servers)) + return zones + + def return_security_group(context, instance_id, security_group_id): pass @@ -80,7 +123,8 @@ def instance_address(context, instance_id): def stub_instance(id, user_id=1, private_address=None, public_addresses=None, - host=None, power_state=0): + host=None, power_state=0, reservation_id="", + uuid=FAKE_UUID): metadata = [] metadata.append(InstanceMetadata(key='seq', value=id)) @@ -92,12 +136,17 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None, if host is not None: host = str(host) + # ReservationID isn't sent back, hack it in there. + server_name = "server%s" % id + if reservation_id != "": + server_name = "reservation_%s" % (reservation_id, ) + instance = { - "id": id, + "id": int(id), "admin_pass": "", "user_id": user_id, "project_id": "", - "image_id": "10", + "image_ref": "10", "kernel_id": "", "ramdisk_id": "", "launch_index": 0, @@ -112,16 +161,17 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None, "host": host, "instance_type": dict(inst_type), "user_data": "", - "reservation_id": "", + "reservation_id": reservation_id, "mac_address": "", - "scheduled_at": datetime.datetime.now(), - "launched_at": datetime.datetime.now(), - "terminated_at": datetime.datetime.now(), + "scheduled_at": utils.utcnow(), + "launched_at": utils.utcnow(), + "terminated_at": utils.utcnow(), "availability_zone": "", - "display_name": "server%s" % id, + "display_name": server_name, "display_description": "", "locked": False, - "metadata": metadata} + "metadata": metadata, + "uuid": uuid} instance["fixed_ip"] = { "address": private_address, @@ -138,6 +188,16 @@ def find_host(self, context, instance_id): return "nova" +class MockSetAdminPassword(object): + def __init__(self): + self.instance_id = None + self.password = None + + def __call__(self, context, instance_id, password): + self.instance_id = instance_id + self.password = password + + class ServersTest(test.TestCase): def setUp(self): @@ -150,8 +210,11 @@ class ServersTest(test.TestCase): fakes.stub_out_auth(self.stubs) fakes.stub_out_key_pair_funcs(self.stubs) fakes.stub_out_image_service(self.stubs) + self.stubs.Set(utils, 'gen_uuid', fake_gen_uuid) self.stubs.Set(nova.db.api, 'instance_get_all', return_servers) - self.stubs.Set(nova.db.api, 'instance_get', return_server) + self.stubs.Set(nova.db.api, 'instance_get', return_server_by_id) + self.stubs.Set(nova.db, 'instance_get_by_uuid', + return_server_by_uuid) self.stubs.Set(nova.db.api, 'instance_get_all_by_user', return_servers) self.stubs.Set(nova.db.api, 'instance_add_security_group', @@ -183,6 +246,36 @@ class ServersTest(test.TestCase): self.assertEqual(res_dict['server']['id'], 1) self.assertEqual(res_dict['server']['name'], 'server1') + def test_get_server_by_uuid(self): + """ + The steps involved with resolving a UUID are pretty complicated; + here's what's happening in this scenario: + + 1. Show is calling `routing_get` + + 2. `routing_get` is wrapped by `reroute_compute` which does the work + of resolving requests to child zones. + + 3. `reroute_compute` looks up the UUID by hitting the stub + (returns_server_by_uuid) + + 4. Since the stub return that the record exists, `reroute_compute` + considers the request to be 'zone local', so it replaces the UUID + in the argument list with an integer ID and then calls the inner + function ('get'). + + 5. The call to `get` hits the other stub 'returns_server_by_id` which + has the UUID set to FAKE_UUID + + So, counterintuitively, we call `get` twice on the `show` command. + """ + req = webob.Request.blank('/v1.0/servers/%s' % FAKE_UUID) + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + self.assertEqual(res_dict['server']['id'], 1) + self.assertEqual(res_dict['server']['uuid'], FAKE_UUID) + self.assertEqual(res_dict['server']['name'], 'server1') + def test_get_server_by_id_v1_1(self): req = webob.Request.blank('/v1.1/servers/1') res = req.get_response(fakes.wsgi_app()) @@ -207,7 +300,6 @@ class ServersTest(test.TestCase): }, ] - print res_dict['server'] self.assertEqual(res_dict['server']['links'], expected_links) def test_get_server_by_id_with_addresses_xml(self): @@ -354,6 +446,57 @@ class ServersTest(test.TestCase): self.assertEqual(s.get('imageId', None), None) i += 1 + def test_get_server_list_with_reservation_id(self): + self.stubs.Set(nova.db.api, 'instance_get_all_by_reservation', + return_servers_by_reservation) + self.stubs.Set(nova.scheduler.api, 'call_zone_method', + return_servers_from_child_zones) + req = webob.Request.blank('/v1.0/servers?reservation_id=foo') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + i = 0 + for s in res_dict['servers']: + if '_is_precooked' in s: + self.assertEqual(s.get('reservation_id'), 'child') + else: + self.assertEqual(s.get('name'), 'server%d' % i) + i += 1 + + def test_get_server_list_with_reservation_id_empty(self): + self.stubs.Set(nova.db.api, 'instance_get_all_by_reservation', + return_servers_by_reservation_empty) + self.stubs.Set(nova.scheduler.api, 'call_zone_method', + return_servers_from_child_zones_empty) + req = webob.Request.blank('/v1.0/servers/detail?reservation_id=foo') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + i = 0 + for s in res_dict['servers']: + if '_is_precooked' in s: + self.assertEqual(s.get('reservation_id'), 'child') + else: + self.assertEqual(s.get('name'), 'server%d' % i) + i += 1 + + def test_get_server_list_with_reservation_id_details(self): + self.stubs.Set(nova.db.api, 'instance_get_all_by_reservation', + return_servers_by_reservation) + self.stubs.Set(nova.scheduler.api, 'call_zone_method', + return_servers_from_child_zones) + req = webob.Request.blank('/v1.0/servers/detail?reservation_id=foo') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + i = 0 + for s in res_dict['servers']: + if '_is_precooked' in s: + self.assertEqual(s.get('reservation_id'), 'child') + else: + self.assertEqual(s.get('name'), 'server%d' % i) + i += 1 + def test_get_server_list_v1_1(self): req = webob.Request.blank('/v1.1/servers') res = req.get_response(fakes.wsgi_app()) @@ -444,7 +587,8 @@ class ServersTest(test.TestCase): def _setup_for_create_instance(self): """Shared implementation for tests below that create instance""" def instance_create(context, inst): - return {'id': '1', 'display_name': 'server_test'} + return {'id': 1, 'display_name': 'server_test', + 'uuid': FAKE_UUID} def server_update(context, id, params): return instance_create(context, id) @@ -473,10 +617,9 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'queue_get_for', queue_get_for) self.stubs.Set(nova.network.manager.VlanManager, 'allocate_fixed_ip', fake_method) - self.stubs.Set(nova.api.openstack.servers.Controller, + self.stubs.Set( + nova.api.openstack.create_instance_helper.CreateInstanceHelper, "_get_kernel_ramdisk_from_image", kernel_ramdisk_mapping) - self.stubs.Set(nova.api.openstack.common, - "get_image_id_from_image_hash", image_id_from_hash) self.stubs.Set(nova.compute.api.API, "_find_host", find_host) def _test_create_instance_helper(self): @@ -499,11 +642,64 @@ class ServersTest(test.TestCase): self.assertEqual(1, server['id']) self.assertEqual(2, server['flavorId']) self.assertEqual(3, server['imageId']) + self.assertEqual(FAKE_UUID, server['uuid']) self.assertEqual(res.status_int, 200) def test_create_instance(self): self._test_create_instance_helper() + def test_create_instance_has_uuid(self): + """Tests at the db-layer instead of API layer since that's where the + UUID is generated + """ + ctxt = context.RequestContext(1, 1) + values = {} + instance = nova.db.api.instance_create(ctxt, values) + expected = FAKE_UUID + self.assertEqual(instance['uuid'], expected) + + def test_create_instance_via_zones(self): + """Server generated ReservationID""" + self._setup_for_create_instance() + FLAGS.allow_admin_api = True + + body = dict(server=dict( + name='server_test', imageId=3, flavorId=2, + metadata={'hello': 'world', 'open': 'stack'}, + personality={})) + req = webob.Request.blank('/v1.0/zones/boot') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + + res = req.get_response(fakes.wsgi_app()) + + reservation_id = json.loads(res.body)['reservation_id'] + self.assertEqual(res.status_int, 200) + self.assertNotEqual(reservation_id, "") + self.assertNotEqual(reservation_id, None) + self.assertTrue(len(reservation_id) > 1) + + def test_create_instance_via_zones_with_resid(self): + """User supplied ReservationID""" + self._setup_for_create_instance() + FLAGS.allow_admin_api = True + + body = dict(server=dict( + name='server_test', imageId=3, flavorId=2, + metadata={'hello': 'world', 'open': 'stack'}, + personality={}, reservation_id='myresid')) + req = webob.Request.blank('/v1.0/zones/boot') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + + res = req.get_response(fakes.wsgi_app()) + + reservation_id = json.loads(res.body)['reservation_id'] + self.assertEqual(res.status_int, 200) + self.assertEqual(reservation_id, "myresid") + def test_create_instance_no_key_pair(self): fakes.stub_out_key_pair_funcs(self.stubs, have_key_pair=False) self._test_create_instance_helper() @@ -579,12 +775,12 @@ class ServersTest(test.TestCase): def test_create_instance_v1_1(self): self._setup_for_create_instance() - image_ref = 'http://localhost/v1.1/images/2' + image_href = 'http://localhost/v1.1/images/2' flavor_ref = 'http://localhost/v1.1/flavors/3' body = { 'server': { 'name': 'server_test', - 'imageRef': image_ref, + 'imageRef': image_href, 'flavorRef': flavor_ref, 'metadata': { 'hello': 'world', @@ -606,16 +802,16 @@ class ServersTest(test.TestCase): self.assertEqual('server_test', server['name']) self.assertEqual(1, server['id']) self.assertEqual(flavor_ref, server['flavorRef']) - self.assertEqual(image_ref, server['imageRef']) + self.assertEqual(image_href, server['imageRef']) self.assertEqual(res.status_int, 200) def test_create_instance_v1_1_bad_href(self): self._setup_for_create_instance() - image_ref = 'http://localhost/v1.1/images/asdf' + image_href = 'http://localhost/v1.1/images/asdf' flavor_ref = 'http://localhost/v1.1/flavors/3' body = dict(server=dict( - name='server_test', imageRef=image_ref, flavorRef=flavor_ref, + name='server_test', imageRef=image_href, flavorRef=flavor_ref, metadata={'hello': 'world', 'open': 'stack'}, personality={})) req = webob.Request.blank('/v1.1/servers') @@ -628,13 +824,12 @@ class ServersTest(test.TestCase): def test_create_instance_v1_1_local_href(self): self._setup_for_create_instance() - image_ref = 'http://localhost/v1.1/images/2' - image_ref_local = '2' + image_id = 2 flavor_ref = 'http://localhost/v1.1/flavors/3' body = { 'server': { 'name': 'server_test', - 'imageRef': image_ref_local, + 'imageRef': image_id, 'flavorRef': flavor_ref, }, } @@ -649,7 +844,7 @@ class ServersTest(test.TestCase): server = json.loads(res.body)['server'] self.assertEqual(1, server['id']) self.assertEqual(flavor_ref, server['flavorRef']) - self.assertEqual(image_ref, server['imageRef']) + self.assertEqual(image_id, server['imageRef']) self.assertEqual(res.status_int, 200) def test_create_instance_with_admin_pass_v1_0(self): @@ -676,12 +871,12 @@ class ServersTest(test.TestCase): def test_create_instance_with_admin_pass_v1_1(self): self._setup_for_create_instance() - image_ref = 'http://localhost/v1.1/images/2' + image_href = 'http://localhost/v1.1/images/2' flavor_ref = 'http://localhost/v1.1/flavors/3' body = { 'server': { 'name': 'server_test', - 'imageRef': image_ref, + 'imageRef': image_href, 'flavorRef': flavor_ref, 'adminPass': 'testpass', }, @@ -698,12 +893,12 @@ class ServersTest(test.TestCase): def test_create_instance_with_empty_admin_pass_v1_1(self): self._setup_for_create_instance() - image_ref = 'http://localhost/v1.1/images/2' + image_href = 'http://localhost/v1.1/images/2' flavor_ref = 'http://localhost/v1.1/flavors/3' body = { 'server': { 'name': 'server_test', - 'imageRef': image_ref, + 'imageRef': image_href, 'flavorRef': flavor_ref, 'adminPass': '', }, @@ -763,16 +958,15 @@ class ServersTest(test.TestCase): self.body = json.dumps(dict(server=inst_dict)) def server_update(context, id, params): - filtered_dict = dict( - display_name='server_test', - admin_pass='bacon', - ) + filtered_dict = dict(display_name='server_test') self.assertEqual(params, filtered_dict) return filtered_dict self.stubs.Set(nova.db.api, 'instance_update', server_update) self.stubs.Set(nova.compute.api.API, "_find_host", find_host) + mock_method = MockSetAdminPassword() + self.stubs.Set(nova.compute.api.API, 'set_admin_password', mock_method) req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' @@ -780,6 +974,8 @@ class ServersTest(test.TestCase): req.body = self.body res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 204) + self.assertEqual(mock_method.instance_id, '1') + self.assertEqual(mock_method.password, 'bacon') def test_update_server_adminPass_ignored_v1_1(self): inst_dict = dict(name='server_test', adminPass='bacon') @@ -831,7 +1027,6 @@ class ServersTest(test.TestCase): req = webob.Request.blank('/v1.0/servers/detail') req.headers['Accept'] = 'application/xml' res = req.get_response(fakes.wsgi_app()) - print res.body dom = minidom.parseString(res.body) for i, server in enumerate(dom.getElementsByTagName('server')): self.assertEqual(server.getAttribute('id'), str(i)) @@ -852,7 +1047,7 @@ class ServersTest(test.TestCase): self.assertEqual(s['id'], i) self.assertEqual(s['hostId'], '') self.assertEqual(s['name'], 'server%d' % i) - self.assertEqual(s['imageId'], '10') + self.assertEqual(s['imageId'], 10) self.assertEqual(s['flavorId'], 1) self.assertEqual(s['status'], 'BUILD') self.assertEqual(s['metadata']['seq'], str(i)) @@ -866,7 +1061,7 @@ class ServersTest(test.TestCase): self.assertEqual(s['id'], i) self.assertEqual(s['hostId'], '') self.assertEqual(s['name'], 'server%d' % i) - self.assertEqual(s['imageRef'], 'http://localhost/v1.1/images/10') + self.assertEqual(s['imageRef'], 10) self.assertEqual(s['flavorRef'], 'http://localhost/v1.1/flavors/1') self.assertEqual(s['status'], 'BUILD') self.assertEqual(s['metadata']['seq'], str(i)) @@ -898,7 +1093,7 @@ class ServersTest(test.TestCase): self.assertEqual(s['id'], i) self.assertEqual(s['hostId'], host_ids[i % 2]) self.assertEqual(s['name'], 'server%d' % i) - self.assertEqual(s['imageId'], '10') + self.assertEqual(s['imageId'], 10) self.assertEqual(s['flavorId'], 1) def test_server_pause(self): @@ -995,17 +1190,15 @@ class ServersTest(test.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 501) - def test_server_change_password_v1_1(self): - - class MockSetAdminPassword(object): - def __init__(self): - self.instance_id = None - self.password = None - - def __call__(self, context, instance_id, password): - self.instance_id = instance_id - self.password = password + def test_server_change_password_xml(self): + req = webob.Request.blank('/v1.0/servers/1/action') + req.method = 'POST' + req.content_type = 'application/xml' + req.body = '<changePassword adminPass="1234pass">' +# res = req.get_response(fakes.wsgi_app()) +# self.assertEqual(res.status_int, 501) + def test_server_change_password_v1_1(self): mock_method = MockSetAdminPassword() self.stubs.Set(nova.compute.api.API, 'set_admin_password', mock_method) body = {'changePassword': {'adminPass': '1234pass'}} @@ -1264,6 +1457,25 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 202) self.assertEqual(self.resize_called, True) + def test_resize_server_v11(self): + + req = webob.Request.blank('/v1.1/servers/1/action') + req.content_type = 'application/json' + req.method = 'POST' + body_dict = dict(resize=dict(flavorRef="http://localhost/3")) + req.body = json.dumps(body_dict) + + self.resize_called = False + + def resize_mock(*args): + self.resize_called = True + + self.stubs.Set(nova.compute.api.API, 'resize', resize_mock) + + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 202) + self.assertEqual(self.resize_called, True) + def test_resize_bad_flavor_fails(self): req = self.webreq('/1/action', 'POST', dict(resize=dict(derp=3))) @@ -1290,7 +1502,7 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 400) def test_resized_server_has_correct_status(self): - req = self.webreq('/1', 'GET', dict(resize=dict(flavorId=3))) + req = self.webreq('/1', 'GET') def fake_migration_get(*args): return {} @@ -1377,13 +1589,13 @@ class ServersTest(test.TestCase): class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def setUp(self): - self.deserializer = servers.ServerCreateRequestXMLDeserializer() + self.deserializer = create_instance_helper.ServerXMLDeserializer() def test_minimal_request(self): serial_request = """ <server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" name="new-server-test" imageId="1" flavorId="1"/>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = {"server": { "name": "new-server-test", "imageId": "1", @@ -1397,7 +1609,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): name="new-server-test" imageId="1" flavorId="1"> <metadata/> </server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = {"server": { "name": "new-server-test", "imageId": "1", @@ -1412,7 +1624,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): name="new-server-test" imageId="1" flavorId="1"> <personality/> </server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = {"server": { "name": "new-server-test", "imageId": "1", @@ -1428,7 +1640,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): <metadata/> <personality/> </server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = {"server": { "name": "new-server-test", "imageId": "1", @@ -1445,7 +1657,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): <personality/> <metadata/> </server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = {"server": { "name": "new-server-test", "imageId": "1", @@ -1463,7 +1675,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): <file path="/etc/conf">aabbccdd</file> </personality> </server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = [{"path": "/etc/conf", "contents": "aabbccdd"}] self.assertEquals(request["server"]["personality"], expected) @@ -1473,7 +1685,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): name="new-server-test" imageId="1" flavorId="1"> <personality><file path="/etc/conf">aabbccdd</file> <file path="/etc/sudoers">abcd</file></personality></server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = [{"path": "/etc/conf", "contents": "aabbccdd"}, {"path": "/etc/sudoers", "contents": "abcd"}] self.assertEquals(request["server"]["personality"], expected) @@ -1489,7 +1701,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): <file path="/etc/ignoreme">anything</file> </personality> </server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = [{"path": "/etc/conf", "contents": "aabbccdd"}] self.assertEquals(request["server"]["personality"], expected) @@ -1498,7 +1710,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): <server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" name="new-server-test" imageId="1" flavorId="1"> <personality><file>aabbccdd</file></personality></server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = [{"contents": "aabbccdd"}] self.assertEquals(request["server"]["personality"], expected) @@ -1507,7 +1719,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): <server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" name="new-server-test" imageId="1" flavorId="1"> <personality><file path="/etc/conf"></file></personality></server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = [{"path": "/etc/conf", "contents": ""}] self.assertEquals(request["server"]["personality"], expected) @@ -1516,7 +1728,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): <server xmlns="http://docs.rackspacecloud.com/servers/api/v1.0" name="new-server-test" imageId="1" flavorId="1"> <personality><file path="/etc/conf"/></personality></server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = [{"path": "/etc/conf", "contents": ""}] self.assertEquals(request["server"]["personality"], expected) @@ -1528,7 +1740,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): <meta key="alpha">beta</meta> </metadata> </server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = {"alpha": "beta"} self.assertEquals(request["server"]["metadata"], expected) @@ -1541,7 +1753,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): <meta key="foo">bar</meta> </metadata> </server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = {"alpha": "beta", "foo": "bar"} self.assertEquals(request["server"]["metadata"], expected) @@ -1553,7 +1765,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): <meta key="alpha"></meta> </metadata> </server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = {"alpha": ""} self.assertEquals(request["server"]["metadata"], expected) @@ -1566,7 +1778,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): <meta key="delta"/> </metadata> </server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = {"alpha": "", "delta": ""} self.assertEquals(request["server"]["metadata"], expected) @@ -1578,7 +1790,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): <meta>beta</meta> </metadata> </server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = {"": "beta"} self.assertEquals(request["server"]["metadata"], expected) @@ -1591,7 +1803,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): <meta>gamma</meta> </metadata> </server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = {"": "gamma"} self.assertEquals(request["server"]["metadata"], expected) @@ -1604,7 +1816,7 @@ class TestServerCreateRequestXMLDeserializer(unittest.TestCase): <meta key="foo">baz</meta> </metadata> </server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') expected = {"foo": "baz"} self.assertEquals(request["server"]["metadata"], expected) @@ -1651,17 +1863,17 @@ b25zLiINCg0KLVJpY2hhcmQgQmFjaA==""", }, ], }} - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') self.assertEqual(request, expected) - def test_request_xmlser_with_flavor_image_ref(self): + def test_request_xmlser_with_flavor_image_href(self): serial_request = """ <server xmlns="http://docs.openstack.org/compute/api/v1.1" name="new-server-test" imageRef="http://localhost:8774/v1.1/images/1" flavorRef="http://localhost:8774/v1.1/flavors/1"> </server>""" - request = self.deserializer.deserialize(serial_request) + request = self.deserializer.deserialize(serial_request, 'create') self.assertEquals(request["server"]["flavorRef"], "http://localhost:8774/v1.1/flavors/1") self.assertEquals(request["server"]["imageRef"], @@ -1676,6 +1888,7 @@ class TestServerInstanceCreation(test.TestCase): fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} fakes.stub_out_auth(self.stubs) + fakes.stub_out_image_service(self.stubs) fakes.stub_out_key_pair_funcs(self.stubs) self.allow_admin = FLAGS.allow_admin_api @@ -1696,7 +1909,8 @@ class TestServerInstanceCreation(test.TestCase): self.injected_files = kwargs['injected_files'] else: self.injected_files = None - return [{'id': '1234', 'display_name': 'fakeinstance'}] + return [{'id': '1234', 'display_name': 'fakeinstance', + 'uuid': FAKE_UUID}] def set_admin_password(self, *args, **kwargs): pass @@ -1708,10 +1922,9 @@ class TestServerInstanceCreation(test.TestCase): compute_api = MockComputeAPI() self.stubs.Set(nova.compute, 'API', make_stub_method(compute_api)) - self.stubs.Set(nova.api.openstack.servers.Controller, + self.stubs.Set( + nova.api.openstack.create_instance_helper.CreateInstanceHelper, '_get_kernel_ramdisk_from_image', make_stub_method((1, 1))) - self.stubs.Set(nova.api.openstack.common, - 'get_image_id_from_image_hash', make_stub_method(2)) return compute_api def _create_personality_request_dict(self, personality_files): @@ -1966,6 +2179,6 @@ class TestGetKernelRamdiskFromImage(test.TestCase): @staticmethod def _get_k_r(image_meta): """Rebinding function to a shorter name for convenience""" - kernel_id, ramdisk_id = \ - servers.Controller._do_get_kernel_ramdisk_from_image(image_meta) + kernel_id, ramdisk_id = create_instance_helper.CreateInstanceHelper. \ + _do_get_kernel_ramdisk_from_image(image_meta) return kernel_id, ramdisk_id diff --git a/nova/tests/api/openstack/test_wsgi.py b/nova/tests/api/openstack/test_wsgi.py new file mode 100644 index 000000000..73a26a087 --- /dev/null +++ b/nova/tests/api/openstack/test_wsgi.py @@ -0,0 +1,305 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +import json +import webob + +from nova import exception +from nova import test +from nova.api.openstack import wsgi + + +class RequestTest(test.TestCase): + def test_content_type_missing(self): + request = wsgi.Request.blank('/tests/123', method='POST') + request.body = "<body />" + self.assertRaises(exception.InvalidContentType, + request.get_content_type) + + def test_content_type_unsupported(self): + request = wsgi.Request.blank('/tests/123', method='POST') + request.headers["Content-Type"] = "text/html" + request.body = "asdf<br />" + self.assertRaises(exception.InvalidContentType, + request.get_content_type) + + def test_content_type_with_charset(self): + request = wsgi.Request.blank('/tests/123') + request.headers["Content-Type"] = "application/json; charset=UTF-8" + result = request.get_content_type() + self.assertEqual(result, "application/json") + + def test_content_type_from_accept_xml(self): + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/xml" + result = request.best_match_content_type() + self.assertEqual(result, "application/xml") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/json" + result = request.best_match_content_type() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/xml, application/json" + result = request.best_match_content_type() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = \ + "application/json; q=0.3, application/xml; q=0.9" + result = request.best_match_content_type() + self.assertEqual(result, "application/xml") + + def test_content_type_from_query_extension(self): + request = wsgi.Request.blank('/tests/123.xml') + result = request.best_match_content_type() + self.assertEqual(result, "application/xml") + + request = wsgi.Request.blank('/tests/123.json') + result = request.best_match_content_type() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123.invalid') + result = request.best_match_content_type() + self.assertEqual(result, "application/json") + + def test_content_type_accept_and_query_extension(self): + request = wsgi.Request.blank('/tests/123.xml') + request.headers["Accept"] = "application/json" + result = request.best_match_content_type() + self.assertEqual(result, "application/xml") + + def test_content_type_accept_default(self): + request = wsgi.Request.blank('/tests/123.unsupported') + request.headers["Accept"] = "application/unsupported1" + result = request.best_match_content_type() + self.assertEqual(result, "application/json") + + +class DictSerializerTest(test.TestCase): + def test_dispatch(self): + serializer = wsgi.DictSerializer() + serializer.create = lambda x: 'pants' + serializer.default = lambda x: 'trousers' + self.assertEqual(serializer.serialize({}, 'create'), 'pants') + + def test_dispatch_default(self): + serializer = wsgi.DictSerializer() + serializer.create = lambda x: 'pants' + serializer.default = lambda x: 'trousers' + self.assertEqual(serializer.serialize({}, 'update'), 'trousers') + + def test_dispatch_action_None(self): + serializer = wsgi.DictSerializer() + serializer.create = lambda x: 'pants' + serializer.default = lambda x: 'trousers' + self.assertEqual(serializer.serialize({}, None), 'trousers') + + +class XMLDictSerializerTest(test.TestCase): + def test_xml(self): + input_dict = dict(servers=dict(a=(2, 3))) + expected_xml = '<serversxmlns="asdf"><a>(2,3)</a></servers>' + serializer = wsgi.XMLDictSerializer(xmlns="asdf") + result = serializer.serialize(input_dict) + result = result.replace('\n', '').replace(' ', '') + self.assertEqual(result, expected_xml) + + +class JSONDictSerializerTest(test.TestCase): + def test_json(self): + input_dict = dict(servers=dict(a=(2, 3))) + expected_json = '{"servers":{"a":[2,3]}}' + serializer = wsgi.JSONDictSerializer() + result = serializer.serialize(input_dict) + result = result.replace('\n', '').replace(' ', '') + self.assertEqual(result, expected_json) + + +class TextDeserializerTest(test.TestCase): + def test_dispatch(self): + deserializer = wsgi.TextDeserializer() + deserializer.create = lambda x: 'pants' + deserializer.default = lambda x: 'trousers' + self.assertEqual(deserializer.deserialize({}, 'create'), 'pants') + + def test_dispatch_default(self): + deserializer = wsgi.TextDeserializer() + deserializer.create = lambda x: 'pants' + deserializer.default = lambda x: 'trousers' + self.assertEqual(deserializer.deserialize({}, 'update'), 'trousers') + + def test_dispatch_action_None(self): + deserializer = wsgi.TextDeserializer() + deserializer.create = lambda x: 'pants' + deserializer.default = lambda x: 'trousers' + self.assertEqual(deserializer.deserialize({}, None), 'trousers') + + +class JSONDeserializerTest(test.TestCase): + def test_json(self): + data = """{"a": { + "a1": "1", + "a2": "2", + "bs": ["1", "2", "3", {"c": {"c1": "1"}}], + "d": {"e": "1"}, + "f": "1"}}""" + as_dict = dict(a={ + 'a1': '1', + 'a2': '2', + 'bs': ['1', '2', '3', {'c': dict(c1='1')}], + 'd': {'e': '1'}, + 'f': '1'}) + deserializer = wsgi.JSONDeserializer() + self.assertEqual(deserializer.deserialize(data), as_dict) + + +class XMLDeserializerTest(test.TestCase): + def test_xml(self): + xml = """ + <a a1="1" a2="2"> + <bs><b>1</b><b>2</b><b>3</b><b><c c1="1"/></b></bs> + <d><e>1</e></d> + <f>1</f> + </a> + """.strip() + as_dict = dict(a={ + 'a1': '1', + 'a2': '2', + 'bs': ['1', '2', '3', {'c': dict(c1='1')}], + 'd': {'e': '1'}, + 'f': '1'}) + metadata = {'plurals': {'bs': 'b', 'ts': 't'}} + deserializer = wsgi.XMLDeserializer(metadata=metadata) + self.assertEqual(deserializer.deserialize(xml), as_dict) + + def test_xml_empty(self): + xml = """<a></a>""" + as_dict = {"a": {}} + deserializer = wsgi.XMLDeserializer() + self.assertEqual(deserializer.deserialize(xml), as_dict) + + +class ResponseSerializerTest(test.TestCase): + def setUp(self): + class JSONSerializer(object): + def serialize(self, data, action='default'): + return 'pew_json' + + class XMLSerializer(object): + def serialize(self, data, action='default'): + return 'pew_xml' + + self.serializers = { + 'application/json': JSONSerializer(), + 'application/XML': XMLSerializer(), + } + + self.serializer = wsgi.ResponseSerializer(serializers=self.serializers) + + def tearDown(self): + pass + + def test_get_serializer(self): + self.assertEqual(self.serializer.get_serializer('application/json'), + self.serializers['application/json']) + + def test_get_serializer_unknown_content_type(self): + self.assertRaises(exception.InvalidContentType, + self.serializer.get_serializer, + 'application/unknown') + + def test_serialize_response(self): + response = self.serializer.serialize({}, 'application/json') + self.assertEqual(response.headers['Content-Type'], 'application/json') + self.assertEqual(response.body, 'pew_json') + + def test_serialize_response_dict_to_unknown_content_type(self): + self.assertRaises(exception.InvalidContentType, + self.serializer.serialize, + {}, 'application/unknown') + + +class RequestDeserializerTest(test.TestCase): + def setUp(self): + class JSONDeserializer(object): + def deserialize(self, data, action='default'): + return 'pew_json' + + class XMLDeserializer(object): + def deserialize(self, data, action='default'): + return 'pew_xml' + + self.deserializers = { + 'application/json': JSONDeserializer(), + 'application/XML': XMLDeserializer(), + } + + self.deserializer = wsgi.RequestDeserializer( + deserializers=self.deserializers) + + def tearDown(self): + pass + + def test_get_deserializer(self): + expected = self.deserializer.get_deserializer('application/json') + self.assertEqual(expected, self.deserializers['application/json']) + + def test_get_deserializer_unknown_content_type(self): + self.assertRaises(exception.InvalidContentType, + self.deserializer.get_deserializer, + 'application/unknown') + + def test_get_expected_content_type(self): + request = wsgi.Request.blank('/') + request.headers['Accept'] = 'application/json' + self.assertEqual(self.deserializer.get_expected_content_type(request), + 'application/json') + + def test_get_action_args(self): + env = { + 'wsgiorg.routing_args': [None, { + 'controller': None, + 'format': None, + 'action': 'update', + 'id': 12, + }], + } + + expected = {'action': 'update', 'id': 12} + + self.assertEqual(self.deserializer.get_action_args(env), expected) + + def test_deserialize(self): + def fake_get_routing_args(request): + return {'action': 'create'} + self.deserializer.get_action_args = fake_get_routing_args + + request = wsgi.Request.blank('/') + request.headers['Accept'] = 'application/xml' + + deserialized = self.deserializer.deserialize(request) + expected = ('create', {}, 'application/xml') + + self.assertEqual(expected, deserialized) + + +class ResourceTest(test.TestCase): + def test_dispatch(self): + class Controller(object): + def index(self, req, pants=None): + return pants + + resource = wsgi.Resource(Controller()) + actual = resource.dispatch(None, 'index', {'pants': 'off'}) + expected = 'off' + self.assertEqual(actual, expected) + + def test_dispatch_unknown_controller_action(self): + class Controller(object): + def index(self, req, pants=None): + return pants + + resource = wsgi.Resource(Controller()) + self.assertRaises(AttributeError, resource.dispatch, + None, 'create', {}) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index fa2e05033..098577e4c 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -21,7 +21,6 @@ import json import nova.db from nova import context from nova import crypto -from nova import exception from nova import flags from nova import test from nova.api.openstack import zones @@ -210,6 +209,11 @@ class ZonesTest(test.TestCase): self.stubs.Set(api, 'select', zone_select) req = webob.Request.blank('/v1.0/zones/select') + req.method = 'POST' + req.headers["Content-Type"] = "application/json" + # Select queries end up being JSON encoded twice. + # Once to a string and again as an HTTP POST Body + req.body = json.dumps(json.dumps({})) res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index 5820ecdc2..d33268296 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -67,192 +67,3 @@ class Test(test.TestCase): self.assertEqual(result.body, "Router result") result = webob.Request.blank('/bad').get_response(Router()) self.assertNotEqual(result.body, "Router result") - - -class ControllerTest(test.TestCase): - - class TestRouter(wsgi.Router): - - class TestController(wsgi.Controller): - - _serialization_metadata = { - 'application/xml': { - "attributes": { - "test": ["id"]}}} - - def show(self, req, id): # pylint: disable=W0622,C0103 - return {"test": {"id": id}} - - def __init__(self): - mapper = routes.Mapper() - mapper.resource("test", "tests", controller=self.TestController()) - wsgi.Router.__init__(self, mapper) - - def test_show(self): - request = wsgi.Request.blank('/tests/123') - result = request.get_response(self.TestRouter()) - self.assertEqual(json.loads(result.body), {"test": {"id": "123"}}) - - def test_response_content_type_from_accept_xml(self): - request = webob.Request.blank('/tests/123') - request.headers["Accept"] = "application/xml" - result = request.get_response(self.TestRouter()) - self.assertEqual(result.headers["Content-Type"], "application/xml") - - def test_response_content_type_from_accept_json(self): - request = wsgi.Request.blank('/tests/123') - request.headers["Accept"] = "application/json" - result = request.get_response(self.TestRouter()) - self.assertEqual(result.headers["Content-Type"], "application/json") - - def test_response_content_type_from_query_extension_xml(self): - request = wsgi.Request.blank('/tests/123.xml') - result = request.get_response(self.TestRouter()) - self.assertEqual(result.headers["Content-Type"], "application/xml") - - def test_response_content_type_from_query_extension_json(self): - request = wsgi.Request.blank('/tests/123.json') - result = request.get_response(self.TestRouter()) - self.assertEqual(result.headers["Content-Type"], "application/json") - - def test_response_content_type_default_when_unsupported(self): - request = wsgi.Request.blank('/tests/123.unsupported') - request.headers["Accept"] = "application/unsupported1" - result = request.get_response(self.TestRouter()) - self.assertEqual(result.status_int, 200) - self.assertEqual(result.headers["Content-Type"], "application/json") - - -class RequestTest(test.TestCase): - - def test_request_content_type_missing(self): - request = wsgi.Request.blank('/tests/123') - request.body = "<body />" - self.assertRaises(webob.exc.HTTPBadRequest, request.get_content_type) - - def test_request_content_type_unsupported(self): - request = wsgi.Request.blank('/tests/123') - request.headers["Content-Type"] = "text/html" - request.body = "asdf<br />" - self.assertRaises(webob.exc.HTTPBadRequest, request.get_content_type) - - def test_request_content_type_with_charset(self): - request = wsgi.Request.blank('/tests/123') - request.headers["Content-Type"] = "application/json; charset=UTF-8" - result = request.get_content_type() - self.assertEqual(result, "application/json") - - def test_content_type_from_accept_xml(self): - request = wsgi.Request.blank('/tests/123') - request.headers["Accept"] = "application/xml" - result = request.best_match_content_type() - self.assertEqual(result, "application/xml") - - request = wsgi.Request.blank('/tests/123') - request.headers["Accept"] = "application/json" - result = request.best_match_content_type() - self.assertEqual(result, "application/json") - - request = wsgi.Request.blank('/tests/123') - request.headers["Accept"] = "application/xml, application/json" - result = request.best_match_content_type() - self.assertEqual(result, "application/json") - - request = wsgi.Request.blank('/tests/123') - request.headers["Accept"] = \ - "application/json; q=0.3, application/xml; q=0.9" - result = request.best_match_content_type() - self.assertEqual(result, "application/xml") - - def test_content_type_from_query_extension(self): - request = wsgi.Request.blank('/tests/123.xml') - result = request.best_match_content_type() - self.assertEqual(result, "application/xml") - - request = wsgi.Request.blank('/tests/123.json') - result = request.best_match_content_type() - self.assertEqual(result, "application/json") - - request = wsgi.Request.blank('/tests/123.invalid') - result = request.best_match_content_type() - self.assertEqual(result, "application/json") - - def test_content_type_accept_and_query_extension(self): - request = wsgi.Request.blank('/tests/123.xml') - request.headers["Accept"] = "application/json" - result = request.best_match_content_type() - self.assertEqual(result, "application/xml") - - def test_content_type_accept_default(self): - request = wsgi.Request.blank('/tests/123.unsupported') - request.headers["Accept"] = "application/unsupported1" - result = request.best_match_content_type() - self.assertEqual(result, "application/json") - - -class SerializerTest(test.TestCase): - - def test_xml(self): - input_dict = dict(servers=dict(a=(2, 3))) - expected_xml = '<servers><a>(2,3)</a></servers>' - serializer = wsgi.Serializer() - result = serializer.serialize(input_dict, "application/xml") - result = result.replace('\n', '').replace(' ', '') - self.assertEqual(result, expected_xml) - - def test_json(self): - input_dict = dict(servers=dict(a=(2, 3))) - expected_json = '{"servers":{"a":[2,3]}}' - serializer = wsgi.Serializer() - result = serializer.serialize(input_dict, "application/json") - result = result.replace('\n', '').replace(' ', '') - self.assertEqual(result, expected_json) - - def test_unsupported_content_type(self): - serializer = wsgi.Serializer() - self.assertRaises(exception.InvalidContentType, serializer.serialize, - {}, "text/null") - - def test_deserialize_json(self): - data = """{"a": { - "a1": "1", - "a2": "2", - "bs": ["1", "2", "3", {"c": {"c1": "1"}}], - "d": {"e": "1"}, - "f": "1"}}""" - as_dict = dict(a={ - 'a1': '1', - 'a2': '2', - 'bs': ['1', '2', '3', {'c': dict(c1='1')}], - 'd': {'e': '1'}, - 'f': '1'}) - metadata = {} - serializer = wsgi.Serializer(metadata) - self.assertEqual(serializer.deserialize(data, "application/json"), - as_dict) - - def test_deserialize_xml(self): - xml = """ - <a a1="1" a2="2"> - <bs><b>1</b><b>2</b><b>3</b><b><c c1="1"/></b></bs> - <d><e>1</e></d> - <f>1</f> - </a> - """.strip() - as_dict = dict(a={ - 'a1': '1', - 'a2': '2', - 'bs': ['1', '2', '3', {'c': dict(c1='1')}], - 'd': {'e': '1'}, - 'f': '1'}) - metadata = {'application/xml': dict(plurals={'bs': 'b', 'ts': 't'})} - serializer = wsgi.Serializer(metadata) - self.assertEqual(serializer.deserialize(xml, "application/xml"), - as_dict) - - def test_deserialize_empty_xml(self): - xml = """<a></a>""" - as_dict = {"a": {}} - serializer = wsgi.Serializer() - self.assertEqual(serializer.deserialize(xml, "application/xml"), - as_dict) diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 5d7ca98b5..2297d2f0e 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -21,24 +21,24 @@ from nova import flags FLAGS = flags.FLAGS flags.DECLARE('volume_driver', 'nova.volume.manager') -FLAGS.volume_driver = 'nova.volume.driver.FakeISCSIDriver' -FLAGS.connection_type = 'fake' -FLAGS.fake_rabbit = True +FLAGS['volume_driver'].SetDefault('nova.volume.driver.FakeISCSIDriver') +FLAGS['connection_type'].SetDefault('fake') +FLAGS['fake_rabbit'].SetDefault(True) flags.DECLARE('auth_driver', 'nova.auth.manager') -FLAGS.auth_driver = 'nova.auth.dbdriver.DbDriver' +FLAGS['auth_driver'].SetDefault('nova.auth.dbdriver.DbDriver') flags.DECLARE('network_size', 'nova.network.manager') flags.DECLARE('num_networks', 'nova.network.manager') flags.DECLARE('fake_network', 'nova.network.manager') -FLAGS.network_size = 8 -FLAGS.num_networks = 2 -FLAGS.fake_network = True -FLAGS.image_service = 'nova.image.local.LocalImageService' +FLAGS['network_size'].SetDefault(8) +FLAGS['num_networks'].SetDefault(2) +FLAGS['fake_network'].SetDefault(True) +FLAGS['image_service'].SetDefault('nova.image.fake.FakeImageService') flags.DECLARE('num_shelves', 'nova.volume.driver') flags.DECLARE('blades_per_shelf', 'nova.volume.driver') flags.DECLARE('iscsi_num_targets', 'nova.volume.driver') -FLAGS.num_shelves = 2 -FLAGS.blades_per_shelf = 4 -FLAGS.iscsi_num_targets = 8 -FLAGS.verbose = True -FLAGS.sqlite_db = "tests.sqlite" -FLAGS.use_ipv6 = True +FLAGS['num_shelves'].SetDefault(2) +FLAGS['blades_per_shelf'].SetDefault(4) +FLAGS['iscsi_num_targets'].SetDefault(8) +FLAGS['verbose'].SetDefault(True) +FLAGS['sqlite_db'].SetDefault("tests.sqlite") +FLAGS['use_ipv6'].SetDefault(True) diff --git a/nova/tests/glance/stubs.py b/nova/tests/glance/stubs.py index 5872552ec..1e0b90d82 100644 --- a/nova/tests/glance/stubs.py +++ b/nova/tests/glance/stubs.py @@ -16,13 +16,14 @@ import StringIO -import glance.client +import nova.image -def stubout_glance_client(stubs, cls): - """Stubs out glance.client.Client""" - stubs.Set(glance.client, 'Client', - lambda *args, **kwargs: cls(*args, **kwargs)) +def stubout_glance_client(stubs): + def fake_get_glance_client(image_href): + image_id = int(str(image_href).split('/')[-1]) + return (FakeGlance('foo'), image_id) + stubs.Set(nova.image, 'get_glance_client', fake_get_glance_client) class FakeGlance(object): diff --git a/nova/tests/image/test_glance.py b/nova/tests/image/test_glance.py index 109905ded..223e7ae57 100644 --- a/nova/tests/image/test_glance.py +++ b/nova/tests/image/test_glance.py @@ -34,7 +34,7 @@ class StubGlanceClient(object): def get_image_meta(self, image_id): return self.images[image_id] - def get_images_detailed(self): + def get_images_detailed(self, filters=None, marker=None, limit=None): return self.images.itervalues() def get_image(self, image_id): @@ -60,10 +60,8 @@ class BaseGlanceTest(unittest.TestCase): NOW_DATETIME = datetime.datetime(2010, 10, 11, 10, 30, 22) def setUp(self): - # FIXME(sirp): we can probably use stubs library here rather than - # dependency injection self.client = StubGlanceClient(None) - self.service = glance.GlanceImageService(self.client) + self.service = glance.GlanceImageService(client=self.client) self.context = context.RequestContext(None, None) def assertDateTimesFilled(self, image_meta): diff --git a/nova/tests/integrated/api/client.py b/nova/tests/integrated/api/client.py index 7e20c9b00..76c03c5fa 100644 --- a/nova/tests/integrated/api/client.py +++ b/nova/tests/integrated/api/client.py @@ -152,7 +152,10 @@ class TestOpenStackClient(object): def _decode_json(self, response): body = response.read() LOG.debug(_("Decoding JSON: %s") % (body)) - return json.loads(body) + if body: + return json.loads(body) + else: + return "" def api_get(self, relative_uri, **kwargs): kwargs.setdefault('check_response_status', [200]) @@ -166,7 +169,7 @@ class TestOpenStackClient(object): headers['Content-Type'] = 'application/json' kwargs['body'] = json.dumps(body) - kwargs.setdefault('check_response_status', [200]) + kwargs.setdefault('check_response_status', [200, 202]) response = self.api_request(relative_uri, **kwargs) return self._decode_json(response) @@ -185,6 +188,9 @@ class TestOpenStackClient(object): def post_server(self, server): return self.api_post('/servers', server)['server'] + def post_server_action(self, server_id, data): + return self.api_post('/servers/%s/action' % server_id, data) + def delete_server(self, server_id): return self.api_delete('/servers/%s' % server_id) @@ -215,30 +221,30 @@ class TestOpenStackClient(object): return self.api_delete('/flavors/%s' % flavor_id) def get_volume(self, volume_id): - return self.api_get('/volumes/%s' % volume_id)['volume'] + return self.api_get('/os-volumes/%s' % volume_id)['volume'] def get_volumes(self, detail=True): - rel_url = '/volumes/detail' if detail else '/volumes' + rel_url = '/os-volumes/detail' if detail else '/os-volumes' return self.api_get(rel_url)['volumes'] def post_volume(self, volume): - return self.api_post('/volumes', volume)['volume'] + return self.api_post('/os-volumes', volume)['volume'] def delete_volume(self, volume_id): - return self.api_delete('/volumes/%s' % volume_id) + return self.api_delete('/os-volumes/%s' % volume_id) def get_server_volume(self, server_id, attachment_id): - return self.api_get('/servers/%s/volume_attachments/%s' % + return self.api_get('/servers/%s/os-volume_attachments/%s' % (server_id, attachment_id))['volumeAttachment'] def get_server_volumes(self, server_id): - return self.api_get('/servers/%s/volume_attachments' % + return self.api_get('/servers/%s/os-volume_attachments' % (server_id))['volumeAttachments'] def post_server_volume(self, server_id, volume_attachment): - return self.api_post('/servers/%s/volume_attachments' % + return self.api_post('/servers/%s/os-volume_attachments' % (server_id), volume_attachment)['volumeAttachment'] def delete_server_volume(self, server_id, attachment_id): - return self.api_delete('/servers/%s/volume_attachments/%s' % + return self.api_delete('/servers/%s/os-volume_attachments/%s' % (server_id, attachment_id)) diff --git a/nova/tests/integrated/integrated_helpers.py b/nova/tests/integrated/integrated_helpers.py index bc98921f0..522c7cb0e 100644 --- a/nova/tests/integrated/integrated_helpers.py +++ b/nova/tests/integrated/integrated_helpers.py @@ -27,6 +27,7 @@ from nova import flags from nova import service from nova import test # For the flags from nova.auth import manager +import nova.image.glance from nova.log import logging from nova.tests.integrated.api import client @@ -151,13 +152,15 @@ class _IntegratedTestBase(test.TestCase): f = self._get_flags() self.flags(**f) + def fake_get_image_service(image_href): + image_id = int(str(image_href).split('/')[-1]) + return (nova.image.fake.FakeImageService(), image_id) + self.stubs.Set(nova.image, 'get_image_service', fake_get_image_service) + # set up services self.start_service('compute') self.start_service('volume') - # NOTE(justinsb): There's a bug here which is eluding me... - # If we start the network_service, all is good, but then subsequent - # tests fail: CloudTestCase.test_ajax_console in particular. - #self.start_service('network') + self.start_service('network') self.start_service('scheduler') self._start_api_service() @@ -202,19 +205,13 @@ class _IntegratedTestBase(test.TestCase): LOG.debug("Image: %s" % image) if 'imageRef' in image: - image_ref = image['imageRef'] + image_href = image['imageRef'] else: - # NOTE(justinsb): The imageRef code hasn't yet landed - LOG.warning("imageRef not yet in images output") - image_ref = image['id'] - - # TODO(justinsb): This is FUBAR - image_ref = abs(hash(image_ref)) - - image_ref = 'http://fake.server/%s' % image_ref + image_href = image['id'] + image_href = 'http://fake.server/%s' % image_href # We now have a valid imageId - server['imageRef'] = image_ref + server['imageRef'] = image_href # Set a valid flavorId flavor = self.api.get_flavors()[0] diff --git a/nova/tests/integrated/test_servers.py b/nova/tests/integrated/test_servers.py index e89d0100a..fcb517cf5 100644 --- a/nova/tests/integrated/test_servers.py +++ b/nova/tests/integrated/test_servers.py @@ -179,6 +179,112 @@ class ServersTest(integrated_helpers._IntegratedTestBase): # Cleanup self._delete_server(created_server_id) + def test_create_and_rebuild_server(self): + """Rebuild a server.""" + + # create a server with initially has no metadata + server = self._build_minimal_create_server_request() + server_post = {'server': server} + created_server = self.api.post_server(server_post) + LOG.debug("created_server: %s" % created_server) + self.assertTrue(created_server['id']) + created_server_id = created_server['id'] + + # rebuild the server with metadata + post = {} + post['rebuild'] = { + "imageRef": "https://localhost/v1.1/32278/images/2", + "name": "blah", + } + + self.api.post_server_action(created_server_id, post) + LOG.debug("rebuilt server: %s" % created_server) + self.assertTrue(created_server['id']) + + found_server = self.api.get_server(created_server_id) + self.assertEqual(created_server_id, found_server['id']) + self.assertEqual({}, found_server.get('metadata')) + self.assertEqual('blah', found_server.get('name')) + + # Cleanup + self._delete_server(created_server_id) + + def test_create_and_rebuild_server_with_metadata(self): + """Rebuild a server with metadata.""" + + # create a server with initially has no metadata + server = self._build_minimal_create_server_request() + server_post = {'server': server} + created_server = self.api.post_server(server_post) + LOG.debug("created_server: %s" % created_server) + self.assertTrue(created_server['id']) + created_server_id = created_server['id'] + + # rebuild the server with metadata + post = {} + post['rebuild'] = { + "imageRef": "https://localhost/v1.1/32278/images/2", + "name": "blah", + } + + metadata = {} + for i in range(30): + metadata['key_%s' % i] = 'value_%s' % i + + post['rebuild']['metadata'] = metadata + + self.api.post_server_action(created_server_id, post) + LOG.debug("rebuilt server: %s" % created_server) + self.assertTrue(created_server['id']) + + found_server = self.api.get_server(created_server_id) + self.assertEqual(created_server_id, found_server['id']) + self.assertEqual(metadata, found_server.get('metadata')) + self.assertEqual('blah', found_server.get('name')) + + # Cleanup + self._delete_server(created_server_id) + + def test_create_and_rebuild_server_with_metadata_removal(self): + """Rebuild a server with metadata.""" + + # create a server with initially has no metadata + server = self._build_minimal_create_server_request() + server_post = {'server': server} + + metadata = {} + for i in range(30): + metadata['key_%s' % i] = 'value_%s' % i + + server_post['server']['metadata'] = metadata + + created_server = self.api.post_server(server_post) + LOG.debug("created_server: %s" % created_server) + self.assertTrue(created_server['id']) + created_server_id = created_server['id'] + + # rebuild the server with metadata + post = {} + post['rebuild'] = { + "imageRef": "https://localhost/v1.1/32278/images/2", + "name": "blah", + } + + metadata = {} + post['rebuild']['metadata'] = metadata + + self.api.post_server_action(created_server_id, post) + LOG.debug("rebuilt server: %s" % created_server) + self.assertTrue(created_server['id']) + + found_server = self.api.get_server(created_server_id) + self.assertEqual(created_server_id, found_server['id']) + self.assertEqual(metadata, found_server.get('metadata')) + self.assertEqual('blah', found_server.get('name')) + + # Cleanup + self._delete_server(created_server_id) + if __name__ == "__main__": unittest.main() diff --git a/nova/tests/integrated/test_xml.py b/nova/tests/integrated/test_xml.py index 8a9754777..fde32f797 100644 --- a/nova/tests/integrated/test_xml.py +++ b/nova/tests/integrated/test_xml.py @@ -32,7 +32,7 @@ class XmlTests(integrated_helpers._IntegratedTestBase): """"Some basic XML sanity checks.""" def test_namespace_limits(self): - """/limits should have v1.0 namespace (hasn't changed in 1.1).""" + """/limits should have v1.1 namespace (has changed in 1.1).""" headers = {} headers['Accept'] = 'application/xml' @@ -40,7 +40,7 @@ class XmlTests(integrated_helpers._IntegratedTestBase): data = response.read() LOG.debug("data: %s" % data) - prefix = '<limits xmlns="%s"' % common.XML_NS_V10 + prefix = '<limits xmlns="%s"' % common.XML_NS_V11 self.assertTrue(data.startswith(prefix)) def test_namespace_servers(self): diff --git a/nova/tests/network/base.py b/nova/tests/network/base.py index b06271c99..f65416824 100644 --- a/nova/tests/network/base.py +++ b/nova/tests/network/base.py @@ -18,7 +18,7 @@ """ Base class of Unit Tests for all network models """ -import IPy +import netaddr import os from nova import context diff --git a/nova/tests/real_flags.py b/nova/tests/real_flags.py deleted file mode 100644 index 71da04992..000000000 --- a/nova/tests/real_flags.py +++ /dev/null @@ -1,26 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from nova import flags - -FLAGS = flags.FLAGS - -FLAGS.connection_type = 'libvirt' -FLAGS.fake_rabbit = False -FLAGS.fake_network = False -FLAGS.verbose = False diff --git a/nova/tests/scheduler/__init__.py b/nova/tests/scheduler/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/nova/tests/scheduler/__init__.py diff --git a/nova/tests/scheduler/test_host_filter.py b/nova/tests/scheduler/test_host_filter.py new file mode 100644 index 000000000..10eafde08 --- /dev/null +++ b/nova/tests/scheduler/test_host_filter.py @@ -0,0 +1,206 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Scheduler Host Filters. +""" + +import json + +from nova import exception +from nova import flags +from nova import test +from nova.scheduler import host_filter + +FLAGS = flags.FLAGS + + +class FakeZoneManager: + pass + + +class HostFilterTestCase(test.TestCase): + """Test case for host filters.""" + + def _host_caps(self, multiplier): + # Returns host capabilities in the following way: + # host1 = memory:free 10 (100max) + # disk:available 100 (1000max) + # hostN = memory:free 10 + 10N + # disk:available 100 + 100N + # in other words: hostN has more resources than host0 + # which means ... don't go above 10 hosts. + return {'host_name-description': 'XenServer %s' % multiplier, + 'host_hostname': 'xs-%s' % multiplier, + 'host_memory_total': 100, + 'host_memory_overhead': 10, + 'host_memory_free': 10 + multiplier * 10, + 'host_memory_free-computed': 10 + multiplier * 10, + 'host_other-config': {}, + 'host_ip_address': '192.168.1.%d' % (100 + multiplier), + 'host_cpu_info': {}, + 'disk_available': 100 + multiplier * 100, + 'disk_total': 1000, + 'disk_used': 0, + 'host_uuid': 'xxx-%d' % multiplier, + 'host_name-label': 'xs-%s' % multiplier} + + def setUp(self): + self.old_flag = FLAGS.default_host_filter + FLAGS.default_host_filter = \ + 'nova.scheduler.host_filter.AllHostsFilter' + self.instance_type = dict(name='tiny', + memory_mb=50, + vcpus=10, + local_gb=500, + flavorid=1, + swap=500, + rxtx_quota=30000, + rxtx_cap=200) + + self.zone_manager = FakeZoneManager() + states = {} + for x in xrange(10): + states['host%02d' % (x + 1)] = {'compute': self._host_caps(x)} + self.zone_manager.service_states = states + + def tearDown(self): + FLAGS.default_host_filter = self.old_flag + + def test_choose_filter(self): + # Test default filter ... + hf = host_filter.choose_host_filter() + self.assertEquals(hf._full_name(), + 'nova.scheduler.host_filter.AllHostsFilter') + # Test valid filter ... + hf = host_filter.choose_host_filter( + 'nova.scheduler.host_filter.InstanceTypeFilter') + self.assertEquals(hf._full_name(), + 'nova.scheduler.host_filter.InstanceTypeFilter') + # Test invalid filter ... + try: + host_filter.choose_host_filter('does not exist') + self.fail("Should not find host filter.") + except exception.SchedulerHostFilterNotFound: + pass + + def test_all_host_filter(self): + hf = host_filter.AllHostsFilter() + cooked = hf.instance_type_to_filter(self.instance_type) + hosts = hf.filter_hosts(self.zone_manager, cooked) + self.assertEquals(10, len(hosts)) + for host, capabilities in hosts: + self.assertTrue(host.startswith('host')) + + def test_instance_type_filter(self): + hf = host_filter.InstanceTypeFilter() + # filter all hosts that can support 50 ram and 500 disk + name, cooked = hf.instance_type_to_filter(self.instance_type) + self.assertEquals('nova.scheduler.host_filter.InstanceTypeFilter', + name) + hosts = hf.filter_hosts(self.zone_manager, cooked) + self.assertEquals(6, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host05', just_hosts[0]) + self.assertEquals('host10', just_hosts[5]) + + def test_json_filter(self): + hf = host_filter.JsonFilter() + # filter all hosts that can support 50 ram and 500 disk + name, cooked = hf.instance_type_to_filter(self.instance_type) + self.assertEquals('nova.scheduler.host_filter.JsonFilter', name) + hosts = hf.filter_hosts(self.zone_manager, cooked) + self.assertEquals(6, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + self.assertEquals('host05', just_hosts[0]) + self.assertEquals('host10', just_hosts[5]) + + # Try some custom queries + + raw = ['or', + ['and', + ['<', '$compute.host_memory_free', 30], + ['<', '$compute.disk_available', 300], + ], + ['and', + ['>', '$compute.host_memory_free', 70], + ['>', '$compute.disk_available', 700], + ] + ] + cooked = json.dumps(raw) + hosts = hf.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(5, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([1, 2, 8, 9, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + + raw = ['not', + ['=', '$compute.host_memory_free', 30], + ] + cooked = json.dumps(raw) + hosts = hf.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(9, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([1, 2, 4, 5, 6, 7, 8, 9, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + + raw = ['in', '$compute.host_memory_free', 20, 40, 60, 80, 100] + cooked = json.dumps(raw) + hosts = hf.filter_hosts(self.zone_manager, cooked) + + self.assertEquals(5, len(hosts)) + just_hosts = [host for host, caps in hosts] + just_hosts.sort() + for index, host in zip([2, 4, 6, 8, 10], just_hosts): + self.assertEquals('host%02d' % index, host) + + # Try some bogus input ... + raw = ['unknown command', ] + cooked = json.dumps(raw) + try: + hf.filter_hosts(self.zone_manager, cooked) + self.fail("Should give KeyError") + except KeyError, e: + pass + + self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps([]))) + self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps({}))) + self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps( + ['not', True, False, True, False], + ))) + + try: + hf.filter_hosts(self.zone_manager, json.dumps( + 'not', True, False, True, False, + )) + self.fail("Should give KeyError") + except KeyError, e: + pass + + self.assertFalse(hf.filter_hosts(self.zone_manager, + json.dumps(['=', '$foo', 100]))) + self.assertFalse(hf.filter_hosts(self.zone_manager, + json.dumps(['=', '$.....', 100]))) + self.assertFalse(hf.filter_hosts(self.zone_manager, + json.dumps( + ['>', ['and', ['or', ['not', ['<', ['>=', ['<=', ['in', ]]]]]]]]))) + + self.assertFalse(hf.filter_hosts(self.zone_manager, + json.dumps(['=', {}, ['>', '$missing....foo']]))) diff --git a/nova/tests/scheduler/test_least_cost_scheduler.py b/nova/tests/scheduler/test_least_cost_scheduler.py new file mode 100644 index 000000000..9a5318aee --- /dev/null +++ b/nova/tests/scheduler/test_least_cost_scheduler.py @@ -0,0 +1,144 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Least Cost Scheduler +""" + +from nova import flags +from nova import test +from nova.scheduler import least_cost +from nova.tests.scheduler import test_zone_aware_scheduler + +MB = 1024 * 1024 +FLAGS = flags.FLAGS + + +class FakeHost(object): + def __init__(self, host_id, free_ram, io): + self.id = host_id + self.free_ram = free_ram + self.io = io + + +class WeightedSumTestCase(test.TestCase): + def test_empty_domain(self): + domain = [] + weighted_fns = [] + result = least_cost.weighted_sum(domain, weighted_fns) + expected = [] + self.assertEqual(expected, result) + + def test_basic_costing(self): + hosts = [ + FakeHost(1, 512 * MB, 100), + FakeHost(2, 256 * MB, 400), + FakeHost(3, 512 * MB, 100), + ] + + weighted_fns = [ + (1, lambda h: h.free_ram), # Fill-first, free_ram is a *cost* + (2, lambda h: h.io), # Avoid high I/O + ] + + costs = least_cost.weighted_sum( + domain=hosts, weighted_fns=weighted_fns) + + # Each 256 MB unit of free-ram contributes 0.5 points by way of: + # cost = weight * (score/max_score) = 1 * (256/512) = 0.5 + # Each 100 iops of IO adds 0.5 points by way of: + # cost = 2 * (100/400) = 2 * 0.25 = 0.5 + expected = [1.5, 2.5, 1.5] + self.assertEqual(expected, costs) + + +class LeastCostSchedulerTestCase(test.TestCase): + def setUp(self): + super(LeastCostSchedulerTestCase, self).setUp() + + class FakeZoneManager: + pass + + zone_manager = FakeZoneManager() + + states = test_zone_aware_scheduler.fake_zone_manager_service_states( + num_hosts=10) + zone_manager.service_states = states + + self.sched = least_cost.LeastCostScheduler() + self.sched.zone_manager = zone_manager + + def tearDown(self): + super(LeastCostSchedulerTestCase, self).tearDown() + + def assertWeights(self, expected, num, request_spec, hosts): + weighted = self.sched.weigh_hosts(num, request_spec, hosts) + self.assertDictListMatch(weighted, expected, approx_equal=True) + + def test_no_hosts(self): + num = 1 + request_spec = {} + hosts = [] + + expected = [] + self.assertWeights(expected, num, request_spec, hosts) + + def test_noop_cost_fn(self): + FLAGS.least_cost_scheduler_cost_functions = [ + 'nova.scheduler.least_cost.noop_cost_fn', + ] + FLAGS.noop_cost_fn_weight = 1 + + num = 1 + request_spec = {} + hosts = self.sched.filter_hosts(num, request_spec) + + expected = [dict(weight=1, hostname=hostname) + for hostname, caps in hosts] + self.assertWeights(expected, num, request_spec, hosts) + + def test_cost_fn_weights(self): + FLAGS.least_cost_scheduler_cost_functions = [ + 'nova.scheduler.least_cost.noop_cost_fn', + ] + FLAGS.noop_cost_fn_weight = 2 + + num = 1 + request_spec = {} + hosts = self.sched.filter_hosts(num, request_spec) + + expected = [dict(weight=2, hostname=hostname) + for hostname, caps in hosts] + self.assertWeights(expected, num, request_spec, hosts) + + def test_fill_first_cost_fn(self): + FLAGS.least_cost_scheduler_cost_functions = [ + 'nova.scheduler.least_cost.fill_first_cost_fn', + ] + FLAGS.fill_first_cost_fn_weight = 1 + + num = 1 + request_spec = {} + hosts = self.sched.filter_hosts(num, request_spec) + + expected = [] + for idx, (hostname, caps) in enumerate(hosts): + # Costs are normalized so over 10 hosts, each host with increasing + # free ram will cost 1/N more. Since the lowest cost host has some + # free ram, we add in the 1/N for the base_cost + weight = 0.1 + (0.1 * idx) + weight_dict = dict(weight=weight, hostname=hostname) + expected.append(weight_dict) + + self.assertWeights(expected, num, request_spec, hosts) diff --git a/nova/tests/test_scheduler.py b/nova/tests/scheduler/test_scheduler.py index 54b3f80fb..4be59d411 100644 --- a/nova/tests/test_scheduler.py +++ b/nova/tests/scheduler/test_scheduler.py @@ -48,6 +48,10 @@ flags.DECLARE('stub_network', 'nova.compute.manager') flags.DECLARE('instances_path', 'nova.compute.manager') +FAKE_UUID_NOT_FOUND = 'ffffffff-ffff-ffff-ffff-ffffffffffff' +FAKE_UUID = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' + + class TestDriver(driver.Scheduler): """Scheduler Driver for Tests""" def schedule(context, topic, *args, **kwargs): @@ -61,7 +65,8 @@ class SchedulerTestCase(test.TestCase): """Test case for scheduler""" def setUp(self): super(SchedulerTestCase, self).setUp() - self.flags(scheduler_driver='nova.tests.test_scheduler.TestDriver') + driver = 'nova.tests.scheduler.test_scheduler.TestDriver' + self.flags(scheduler_driver=driver) def _create_compute_service(self): """Create compute-manager(ComputeNode and Service record).""" @@ -196,7 +201,7 @@ class ZoneSchedulerTestCase(test.TestCase): service.topic = 'compute' service.id = kwargs['id'] service.availability_zone = kwargs['zone'] - service.created_at = datetime.datetime.utcnow() + service.created_at = utils.utcnow() return service def test_with_two_zones(self): @@ -290,7 +295,7 @@ class SimpleDriverTestCase(test.TestCase): dic['host'] = kwargs.get('host', 'dummy') s_ref = db.service_create(self.context, dic) if 'created_at' in kwargs.keys() or 'updated_at' in kwargs.keys(): - t = datetime.datetime.utcnow() - datetime.timedelta(0) + t = utils.utcnow() - datetime.timedelta(0) dic['created_at'] = kwargs.get('created_at', t) dic['updated_at'] = kwargs.get('updated_at', t) db.service_update(self.context, s_ref['id'], dic) @@ -401,7 +406,7 @@ class SimpleDriverTestCase(test.TestCase): FLAGS.compute_manager) compute1.start() s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') - now = datetime.datetime.utcnow() + now = utils.utcnow() delta = datetime.timedelta(seconds=FLAGS.service_down_time * 2) past = now - delta db.service_update(self.context, s1['id'], {'updated_at': past}) @@ -542,7 +547,7 @@ class SimpleDriverTestCase(test.TestCase): def test_wont_sechedule_if_specified_host_is_down(self): compute1 = self.start_service('compute', host='host1') s1 = db.service_get_by_args(self.context, 'host1', 'nova-compute') - now = datetime.datetime.utcnow() + now = utils.utcnow() delta = datetime.timedelta(seconds=FLAGS.service_down_time * 2) past = now - delta db.service_update(self.context, s1['id'], {'updated_at': past}) @@ -692,7 +697,7 @@ class SimpleDriverTestCase(test.TestCase): dic = {'instance_id': instance_id, 'size': 1} v_ref = db.volume_create(self.context, {'instance_id': instance_id, 'size': 1}) - t1 = datetime.datetime.utcnow() - datetime.timedelta(1) + t1 = utils.utcnow() - datetime.timedelta(1) dic = {'created_at': t1, 'updated_at': t1, 'binary': 'nova-volume', 'topic': 'volume', 'report_count': 0} s_ref = db.service_create(self.context, dic) @@ -709,7 +714,7 @@ class SimpleDriverTestCase(test.TestCase): """Confirms src-compute node is alive.""" instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) - t = datetime.datetime.utcnow() - datetime.timedelta(10) + t = utils.utcnow() - datetime.timedelta(10) s_ref = self._create_compute_service(created_at=t, updated_at=t, host=i_ref['host']) @@ -737,7 +742,7 @@ class SimpleDriverTestCase(test.TestCase): """Confirms exception raises in case dest host does not exist.""" instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) - t = datetime.datetime.utcnow() - datetime.timedelta(10) + t = utils.utcnow() - datetime.timedelta(10) s_ref = self._create_compute_service(created_at=t, updated_at=t, host=i_ref['host']) @@ -796,7 +801,7 @@ class SimpleDriverTestCase(test.TestCase): # mocks for live_migration_common_check() instance_id = self._create_instance() i_ref = db.instance_get(self.context, instance_id) - t1 = datetime.datetime.utcnow() - datetime.timedelta(10) + t1 = utils.utcnow() - datetime.timedelta(10) s_ref = self._create_compute_service(created_at=t1, updated_at=t1, host=dest) @@ -925,12 +930,23 @@ def zone_get_all(context): ] +def fake_instance_get_by_uuid(context, uuid): + if FAKE_UUID_NOT_FOUND: + raise exception.InstanceNotFound(instance_id=uuid) + else: + return {'id': 1} + + class FakeRerouteCompute(api.reroute_compute): + def __init__(self, method_name, id_to_return=1): + super(FakeRerouteCompute, self).__init__(method_name) + self.id_to_return = id_to_return + def _call_child_zones(self, zones, function): return [] def get_collection_context_and_id(self, args, kwargs): - return ("servers", None, 1) + return ("servers", None, self.id_to_return) def unmarshall_result(self, zone_responses): return dict(magic="found me") @@ -959,6 +975,8 @@ class ZoneRedirectTest(test.TestCase): self.stubs = stubout.StubOutForTesting() self.stubs.Set(db, 'zone_get_all', zone_get_all) + self.stubs.Set(db, 'instance_get_by_uuid', + fake_instance_get_by_uuid) self.enable_zone_routing = FLAGS.enable_zone_routing FLAGS.enable_zone_routing = True @@ -975,8 +993,19 @@ class ZoneRedirectTest(test.TestCase): except api.RedirectResult, e: self.fail(_("Successful database hit should succeed")) - def test_trap_not_found_locally(self): + def test_trap_not_found_locally_id_passed(self): + """When an integer ID is not found locally, we cannot reroute to + another zone, so just return InstanceNotFound exception + """ decorator = FakeRerouteCompute("foo") + self.assertRaises(exception.InstanceNotFound, + decorator(go_boom), None, None, 1) + + def test_trap_not_found_locally_uuid_passed(self): + """When a UUID is found, if the item isn't found locally, we should + try to reroute to a child zone to see if they have it + """ + decorator = FakeRerouteCompute("foo", id_to_return=FAKE_UUID_NOT_FOUND) try: result = decorator(go_boom)(None, None, 1) self.assertFail(_("Should have rerouted.")) @@ -1109,10 +1138,4 @@ class CallZoneMethodTest(test.TestCase): def test_call_zone_method_generates_exception(self): context = {} method = 'raises_exception' - results = api.call_zone_method(context, method) - - # FIXME(sirp): for now the _error_trap code is catching errors and - # converting them to a ("ERROR", "string") tuples. The code (and this - # test) should eventually handle real exceptions. - expected = [(1, ('ERROR', 'testing'))] - self.assertEqual(expected, results) + self.assertRaises(Exception, api.call_zone_method, context, method) diff --git a/nova/tests/scheduler/test_zone_aware_scheduler.py b/nova/tests/scheduler/test_zone_aware_scheduler.py new file mode 100644 index 000000000..37c6488cc --- /dev/null +++ b/nova/tests/scheduler/test_zone_aware_scheduler.py @@ -0,0 +1,296 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Tests For Zone Aware Scheduler. +""" + +from nova import exception +from nova import test +from nova.scheduler import driver +from nova.scheduler import zone_aware_scheduler +from nova.scheduler import zone_manager + + +def _host_caps(multiplier): + # Returns host capabilities in the following way: + # host1 = memory:free 10 (100max) + # disk:available 100 (1000max) + # hostN = memory:free 10 + 10N + # disk:available 100 + 100N + # in other words: hostN has more resources than host0 + # which means ... don't go above 10 hosts. + return {'host_name-description': 'XenServer %s' % multiplier, + 'host_hostname': 'xs-%s' % multiplier, + 'host_memory_total': 100, + 'host_memory_overhead': 10, + 'host_memory_free': 10 + multiplier * 10, + 'host_memory_free-computed': 10 + multiplier * 10, + 'host_other-config': {}, + 'host_ip_address': '192.168.1.%d' % (100 + multiplier), + 'host_cpu_info': {}, + 'disk_available': 100 + multiplier * 100, + 'disk_total': 1000, + 'disk_used': 0, + 'host_uuid': 'xxx-%d' % multiplier, + 'host_name-label': 'xs-%s' % multiplier} + + +def fake_zone_manager_service_states(num_hosts): + states = {} + for x in xrange(num_hosts): + states['host%02d' % (x + 1)] = {'compute': _host_caps(x)} + return states + + +class FakeZoneAwareScheduler(zone_aware_scheduler.ZoneAwareScheduler): + def filter_hosts(self, num, specs): + # NOTE(sirp): this is returning [(hostname, services)] + return self.zone_manager.service_states.items() + + def weigh_hosts(self, num, specs, hosts): + fake_weight = 99 + weighted = [] + for hostname, caps in hosts: + weighted.append(dict(weight=fake_weight, name=hostname)) + return weighted + + +class FakeZoneManager(zone_manager.ZoneManager): + def __init__(self): + self.service_states = { + 'host1': { + 'compute': {'ram': 1000}, + }, + 'host2': { + 'compute': {'ram': 2000}, + }, + 'host3': { + 'compute': {'ram': 3000}, + }, + } + + +class FakeEmptyZoneManager(zone_manager.ZoneManager): + def __init__(self): + self.service_states = {} + + +def fake_empty_call_zone_method(context, method, specs): + return [] + + +# Hmm, I should probably be using mox for this. +was_called = False + + +def fake_provision_resource(context, item, instance_id, request_spec, kwargs): + global was_called + was_called = True + + +def fake_ask_child_zone_to_create_instance(context, zone_info, + request_spec, kwargs): + global was_called + was_called = True + + +def fake_provision_resource_locally(context, item, instance_id, kwargs): + global was_called + was_called = True + + +def fake_provision_resource_from_blob(context, item, instance_id, + request_spec, kwargs): + global was_called + was_called = True + + +def fake_decrypt_blob_returns_local_info(blob): + return {'foo': True} # values aren't important. + + +def fake_decrypt_blob_returns_child_info(blob): + return {'child_zone': True, + 'child_blob': True} # values aren't important. Keys are. + + +def fake_call_zone_method(context, method, specs): + return [ + ('zone1', [ + dict(weight=1, blob='AAAAAAA'), + dict(weight=111, blob='BBBBBBB'), + dict(weight=112, blob='CCCCCCC'), + dict(weight=113, blob='DDDDDDD'), + ]), + ('zone2', [ + dict(weight=120, blob='EEEEEEE'), + dict(weight=2, blob='FFFFFFF'), + dict(weight=122, blob='GGGGGGG'), + dict(weight=123, blob='HHHHHHH'), + ]), + ('zone3', [ + dict(weight=130, blob='IIIIIII'), + dict(weight=131, blob='JJJJJJJ'), + dict(weight=132, blob='KKKKKKK'), + dict(weight=3, blob='LLLLLLL'), + ]), + ] + + +class ZoneAwareSchedulerTestCase(test.TestCase): + """Test case for Zone Aware Scheduler.""" + + def test_zone_aware_scheduler(self): + """ + Create a nested set of FakeZones, ensure that a select call returns the + appropriate build plan. + """ + sched = FakeZoneAwareScheduler() + self.stubs.Set(sched, '_call_zone_method', fake_call_zone_method) + + zm = FakeZoneManager() + sched.set_zone_manager(zm) + + fake_context = {} + build_plan = sched.select(fake_context, {}) + + self.assertEqual(15, len(build_plan)) + + hostnames = [plan_item['name'] + for plan_item in build_plan if 'name' in plan_item] + self.assertEqual(3, len(hostnames)) + + def test_empty_zone_aware_scheduler(self): + """ + Ensure empty hosts & child_zones result in NoValidHosts exception. + """ + sched = FakeZoneAwareScheduler() + self.stubs.Set(sched, '_call_zone_method', fake_empty_call_zone_method) + + zm = FakeEmptyZoneManager() + sched.set_zone_manager(zm) + + fake_context = {} + self.assertRaises(driver.NoValidHost, sched.schedule_run_instance, + fake_context, 1, + dict(host_filter=None, + request_spec={'instance_type': {}})) + + def test_schedule_do_not_schedule_with_hint(self): + """ + Check the local/child zone routing in the run_instance() call. + If the zone_blob hint was passed in, don't re-schedule. + """ + global was_called + sched = FakeZoneAwareScheduler() + was_called = False + self.stubs.Set(sched, '_provision_resource', fake_provision_resource) + request_spec = { + 'instance_properties': {}, + 'instance_type': {}, + 'filter_driver': 'nova.scheduler.host_filter.AllHostsFilter', + 'blob': "Non-None blob data", + } + + result = sched.schedule_run_instance(None, 1, request_spec) + self.assertEquals(None, result) + self.assertTrue(was_called) + + def test_provision_resource_local(self): + """Provision a resource locally or remotely.""" + global was_called + sched = FakeZoneAwareScheduler() + was_called = False + self.stubs.Set(sched, '_provision_resource_locally', + fake_provision_resource_locally) + + request_spec = {'hostname': "foo"} + sched._provision_resource(None, request_spec, 1, request_spec, {}) + self.assertTrue(was_called) + + def test_provision_resource_remote(self): + """Provision a resource locally or remotely.""" + global was_called + sched = FakeZoneAwareScheduler() + was_called = False + self.stubs.Set(sched, '_provision_resource_from_blob', + fake_provision_resource_from_blob) + + request_spec = {} + sched._provision_resource(None, request_spec, 1, request_spec, {}) + self.assertTrue(was_called) + + def test_provision_resource_from_blob_empty(self): + """Provision a resource locally or remotely given no hints.""" + global was_called + sched = FakeZoneAwareScheduler() + request_spec = {} + self.assertRaises(zone_aware_scheduler.InvalidBlob, + sched._provision_resource_from_blob, + None, {}, 1, {}, {}) + + def test_provision_resource_from_blob_with_local_blob(self): + """ + Provision a resource locally or remotely when blob hint passed in. + """ + global was_called + sched = FakeZoneAwareScheduler() + was_called = False + self.stubs.Set(sched, '_decrypt_blob', + fake_decrypt_blob_returns_local_info) + self.stubs.Set(sched, '_provision_resource_locally', + fake_provision_resource_locally) + + request_spec = {'blob': "Non-None blob data"} + + sched._provision_resource_from_blob(None, request_spec, 1, + request_spec, {}) + self.assertTrue(was_called) + + def test_provision_resource_from_blob_with_child_blob(self): + """ + Provision a resource locally or remotely when child blob hint + passed in. + """ + global was_called + sched = FakeZoneAwareScheduler() + self.stubs.Set(sched, '_decrypt_blob', + fake_decrypt_blob_returns_child_info) + was_called = False + self.stubs.Set(sched, '_ask_child_zone_to_create_instance', + fake_ask_child_zone_to_create_instance) + + request_spec = {'blob': "Non-None blob data"} + + sched._provision_resource_from_blob(None, request_spec, 1, + request_spec, {}) + self.assertTrue(was_called) + + def test_provision_resource_from_blob_with_immediate_child_blob(self): + """ + Provision a resource locally or remotely when blob hint passed in + from an immediate child. + """ + global was_called + sched = FakeZoneAwareScheduler() + was_called = False + self.stubs.Set(sched, '_ask_child_zone_to_create_instance', + fake_ask_child_zone_to_create_instance) + + request_spec = {'child_blob': True, 'child_zone': True} + + sched._provision_resource_from_blob(None, request_spec, 1, + request_spec, {}) + self.assertTrue(was_called) diff --git a/nova/tests/test_adminapi.py b/nova/tests/test_adminapi.py new file mode 100644 index 000000000..ce826fd5b --- /dev/null +++ b/nova/tests/test_adminapi.py @@ -0,0 +1,111 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from eventlet import greenthread + +from nova import context +from nova import db +from nova import flags +from nova import log as logging +from nova import rpc +from nova import test +from nova import utils +from nova.auth import manager +from nova.api.ec2 import admin +from nova.image import fake + + +FLAGS = flags.FLAGS +LOG = logging.getLogger('nova.tests.adminapi') + + +class AdminApiTestCase(test.TestCase): + def setUp(self): + super(AdminApiTestCase, self).setUp() + self.flags(connection_type='fake') + + self.conn = rpc.Connection.instance() + + # set up our cloud + self.api = admin.AdminController() + + # set up services + self.compute = self.start_service('compute') + self.scheduter = self.start_service('scheduler') + self.network = self.start_service('network') + self.volume = self.start_service('volume') + self.image_service = utils.import_object(FLAGS.image_service) + + self.manager = manager.AuthManager() + self.user = self.manager.create_user('admin', 'admin', 'admin', True) + self.project = self.manager.create_project('proj', 'admin', 'proj') + self.context = context.RequestContext(user=self.user, + project=self.project) + host = self.network.get_network_host(self.context.elevated()) + + def fake_show(meh, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, + 'type': 'machine', 'image_state': 'available'}} + + self.stubs.Set(fake._FakeImageService, 'show', fake_show) + self.stubs.Set(fake._FakeImageService, 'show_by_name', fake_show) + + # NOTE(vish): set up a manual wait so rpc.cast has a chance to finish + rpc_cast = rpc.cast + + def finish_cast(*args, **kwargs): + rpc_cast(*args, **kwargs) + greenthread.sleep(0.2) + + self.stubs.Set(rpc, 'cast', finish_cast) + + def tearDown(self): + network_ref = db.project_get_network(self.context, + self.project.id) + db.network_disassociate(self.context, network_ref['id']) + self.manager.delete_project(self.project) + self.manager.delete_user(self.user) + super(AdminApiTestCase, self).tearDown() + + def test_block_external_ips(self): + """Make sure provider firewall rules are created.""" + result = self.api.block_external_addresses(self.context, '1.1.1.1/32') + self.api.remove_external_address_block(self.context, '1.1.1.1/32') + self.assertEqual('OK', result['status']) + self.assertEqual('Added 3 rules', result['message']) + + def test_list_blocked_ips(self): + """Make sure we can see the external blocks that exist.""" + self.api.block_external_addresses(self.context, '1.1.1.2/32') + result = self.api.describe_external_address_blocks(self.context) + num = len(db.provider_fw_rule_get_all(self.context)) + self.api.remove_external_address_block(self.context, '1.1.1.2/32') + # we only list IP, not tcp/udp/icmp rules + self.assertEqual(num / 3, len(result['externalIpBlockInfo'])) + + def test_remove_ip_block(self): + """Remove ip blocks.""" + result = self.api.block_external_addresses(self.context, '1.1.1.3/32') + self.assertEqual('OK', result['status']) + num0 = len(db.provider_fw_rule_get_all(self.context)) + result = self.api.remove_external_address_block(self.context, + '1.1.1.3/32') + self.assertEqual('OK', result['status']) + self.assertEqual('Deleted 3 rules', result['message']) + num1 = len(db.provider_fw_rule_get_all(self.context)) + self.assert_(num1 < num0) diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index 97f401b87..20b20fcbf 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -89,7 +89,7 @@ class FakeHttplibConnection(object): class XmlConversionTestCase(test.TestCase): """Unit test api xml conversion""" def test_number_conversion(self): - conv = apirequest._try_convert + conv = ec2utils._try_convert self.assertEqual(conv('None'), None) self.assertEqual(conv('True'), True) self.assertEqual(conv('False'), False) @@ -224,6 +224,29 @@ class ApiEc2TestCase(test.TestCase): self.manager.delete_project(project) self.manager.delete_user(user) + def test_create_duplicate_key_pair(self): + """Test that, after successfully generating a keypair, + requesting a second keypair with the same name fails sanely""" + self.expect_http() + self.mox.ReplayAll() + keyname = "".join(random.choice("sdiuisudfsdcnpaqwertasd") \ + for x in range(random.randint(4, 8))) + user = self.manager.create_user('fake', 'fake', 'fake') + project = self.manager.create_project('fake', 'fake', 'fake') + # NOTE(vish): create depends on pool, so call helper directly + self.ec2.create_key_pair('test') + + try: + self.ec2.create_key_pair('test') + except EC2ResponseError, e: + if e.code == 'KeyPairExists': + pass + else: + self.fail("Unexpected EC2ResponseError: %s " + "(expected KeyPairExists)" % e.code) + else: + self.fail('Exception not raised.') + def test_get_all_security_groups(self): """Test that we can retrieve security groups""" self.expect_http() diff --git a/nova/tests/test_auth.py b/nova/tests/test_auth.py index f02dd94b7..7d00bddfe 100644 --- a/nova/tests/test_auth.py +++ b/nova/tests/test_auth.py @@ -86,6 +86,7 @@ class _AuthManagerBaseTestCase(test.TestCase): super(_AuthManagerBaseTestCase, self).setUp() self.flags(connection_type='fake') self.manager = manager.AuthManager(new=True) + self.manager.mc.cache = {} def test_create_and_find_user(self): with user_generator(self.manager): diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 54c0454de..6327734f5 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -17,32 +17,25 @@ # under the License. from base64 import b64decode -import json from M2Crypto import BIO from M2Crypto import RSA import os -import shutil -import tempfile -import time from eventlet import greenthread from nova import context from nova import crypto from nova import db +from nova import exception from nova import flags from nova import log as logging from nova import rpc -from nova import service from nova import test from nova import utils -from nova import exception from nova.auth import manager -from nova.compute import power_state from nova.api.ec2 import cloud from nova.api.ec2 import ec2utils -from nova.image import local -from nova.exception import NotFound +from nova.image import fake FLAGS = flags.FLAGS @@ -63,6 +56,7 @@ class CloudTestCase(test.TestCase): self.compute = self.start_service('compute') self.scheduter = self.start_service('scheduler') self.network = self.start_service('network') + self.volume = self.start_service('volume') self.image_service = utils.import_object(FLAGS.image_service) self.manager = manager.AuthManager() @@ -74,10 +68,19 @@ class CloudTestCase(test.TestCase): def fake_show(meh, context, id): return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, - 'type': 'machine'}} + 'type': 'machine', 'image_state': 'available'}} + + self.stubs.Set(fake._FakeImageService, 'show', fake_show) + self.stubs.Set(fake._FakeImageService, 'show_by_name', fake_show) - self.stubs.Set(local.LocalImageService, 'show', fake_show) - self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show) + # NOTE(vish): set up a manual wait so rpc.cast has a chance to finish + rpc_cast = rpc.cast + + def finish_cast(*args, **kwargs): + rpc_cast(*args, **kwargs) + greenthread.sleep(0.2) + + self.stubs.Set(rpc, 'cast', finish_cast) def tearDown(self): network_ref = db.project_get_network(self.context, @@ -85,8 +88,6 @@ class CloudTestCase(test.TestCase): db.network_disassociate(self.context, network_ref['id']) self.manager.delete_project(self.project) self.manager.delete_user(self.user) - self.compute.kill() - self.network.kill() super(CloudTestCase, self).tearDown() def _create_key(self, name): @@ -113,9 +114,20 @@ class CloudTestCase(test.TestCase): self.cloud.describe_addresses(self.context) self.cloud.release_address(self.context, public_ip=address) - greenthread.sleep(0.3) db.floating_ip_destroy(self.context, address) + def test_allocate_address(self): + address = "10.10.10.10" + allocate = self.cloud.allocate_address + db.floating_ip_create(self.context, + {'address': address, + 'host': self.network.host}) + self.assertEqual(allocate(self.context)['publicIp'], address) + db.floating_ip_destroy(self.context, address) + self.assertRaises(exception.NoMoreFloatingIps, + allocate, + self.context) + def test_associate_disassociate_address(self): """Verifies associate runs cleanly without raising an exception""" address = "10.10.10.10" @@ -129,12 +141,10 @@ class CloudTestCase(test.TestCase): self.cloud.associate_address(self.context, instance_id=ec2_id, public_ip=address) - greenthread.sleep(0.3) self.cloud.disassociate_address(self.context, public_ip=address) self.cloud.release_address(self.context, public_ip=address) - greenthread.sleep(0.3) self.network.deallocate_fixed_ip(self.context, fixed) db.instance_destroy(self.context, inst['id']) db.floating_ip_destroy(self.context, address) @@ -171,6 +181,25 @@ class CloudTestCase(test.TestCase): db.volume_destroy(self.context, vol1['id']) db.volume_destroy(self.context, vol2['id']) + def test_create_volume_from_snapshot(self): + """Makes sure create_volume works when we specify a snapshot.""" + vol = db.volume_create(self.context, {'size': 1}) + snap = db.snapshot_create(self.context, {'volume_id': vol['id'], + 'volume_size': vol['size'], + 'status': "available"}) + snapshot_id = ec2utils.id_to_ec2_id(snap['id'], 'snap-%08x') + + result = self.cloud.create_volume(self.context, + snapshot_id=snapshot_id) + volume_id = result['volumeId'] + result = self.cloud.describe_volumes(self.context) + self.assertEqual(len(result['volumeSet']), 2) + self.assertEqual(result['volumeSet'][1]['volumeId'], volume_id) + + db.volume_destroy(self.context, ec2utils.ec2_id_to_id(volume_id)) + db.snapshot_destroy(self.context, snap['id']) + db.volume_destroy(self.context, vol['id']) + def test_describe_availability_zones(self): """Makes sure describe_availability_zones works and filters results.""" service1 = db.service_create(self.context, {'host': 'host1_zones', @@ -188,13 +217,59 @@ class CloudTestCase(test.TestCase): db.service_destroy(self.context, service1['id']) db.service_destroy(self.context, service2['id']) + def test_describe_snapshots(self): + """Makes sure describe_snapshots works and filters results.""" + vol = db.volume_create(self.context, {}) + snap1 = db.snapshot_create(self.context, {'volume_id': vol['id']}) + snap2 = db.snapshot_create(self.context, {'volume_id': vol['id']}) + result = self.cloud.describe_snapshots(self.context) + self.assertEqual(len(result['snapshotSet']), 2) + snapshot_id = ec2utils.id_to_ec2_id(snap2['id'], 'snap-%08x') + result = self.cloud.describe_snapshots(self.context, + snapshot_id=[snapshot_id]) + self.assertEqual(len(result['snapshotSet']), 1) + self.assertEqual( + ec2utils.ec2_id_to_id(result['snapshotSet'][0]['snapshotId']), + snap2['id']) + db.snapshot_destroy(self.context, snap1['id']) + db.snapshot_destroy(self.context, snap2['id']) + db.volume_destroy(self.context, vol['id']) + + def test_create_snapshot(self): + """Makes sure create_snapshot works.""" + vol = db.volume_create(self.context, {'status': "available"}) + volume_id = ec2utils.id_to_ec2_id(vol['id'], 'vol-%08x') + + result = self.cloud.create_snapshot(self.context, + volume_id=volume_id) + snapshot_id = result['snapshotId'] + result = self.cloud.describe_snapshots(self.context) + self.assertEqual(len(result['snapshotSet']), 1) + self.assertEqual(result['snapshotSet'][0]['snapshotId'], snapshot_id) + + db.snapshot_destroy(self.context, ec2utils.ec2_id_to_id(snapshot_id)) + db.volume_destroy(self.context, vol['id']) + + def test_delete_snapshot(self): + """Makes sure delete_snapshot works.""" + vol = db.volume_create(self.context, {'status': "available"}) + snap = db.snapshot_create(self.context, {'volume_id': vol['id'], + 'status': "available"}) + snapshot_id = ec2utils.id_to_ec2_id(snap['id'], 'snap-%08x') + + result = self.cloud.delete_snapshot(self.context, + snapshot_id=snapshot_id) + self.assertTrue(result) + + db.volume_destroy(self.context, vol['id']) + def test_describe_instances(self): """Makes sure describe_instances works and filters results.""" inst1 = db.instance_create(self.context, {'reservation_id': 'a', - 'image_id': 1, + 'image_ref': 1, 'host': 'host1'}) inst2 = db.instance_create(self.context, {'reservation_id': 'a', - 'image_id': 1, + 'image_ref': 1, 'host': 'host2'}) comp1 = db.service_create(self.context, {'host': 'host1', 'availability_zone': 'zone1', @@ -227,9 +302,9 @@ class CloudTestCase(test.TestCase): 'type': 'machine'}}] def fake_show_none(meh, context, id): - raise NotFound + raise exception.ImageNotFound(image_id='bad_image_id') - self.stubs.Set(local.LocalImageService, 'detail', fake_detail) + self.stubs.Set(fake._FakeImageService, 'detail', fake_detail) # list all result1 = describe_images(self.context) result1 = result1['imagesSet'][0] @@ -243,9 +318,9 @@ class CloudTestCase(test.TestCase): self.assertEqual(2, len(result3['imagesSet'])) # provide an non-existing image_id self.stubs.UnsetAll() - self.stubs.Set(local.LocalImageService, 'show', fake_show_none) - self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show_none) - self.assertRaises(NotFound, describe_images, + self.stubs.Set(fake._FakeImageService, 'show', fake_show_none) + self.stubs.Set(fake._FakeImageService, 'show_by_name', fake_show_none) + self.assertRaises(exception.ImageNotFound, describe_images, self.context, ['ami-fake']) def test_describe_image_attribute(self): @@ -255,8 +330,8 @@ class CloudTestCase(test.TestCase): return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, 'type': 'machine'}, 'is_public': True} - self.stubs.Set(local.LocalImageService, 'show', fake_show) - self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show) + self.stubs.Set(fake._FakeImageService, 'show', fake_show) + self.stubs.Set(fake._FakeImageService, 'show_by_name', fake_show) result = describe_image_attribute(self.context, 'ami-00000001', 'launchPermission') self.assertEqual([{'group': 'all'}], result['launchPermission']) @@ -271,9 +346,9 @@ class CloudTestCase(test.TestCase): def fake_update(meh, context, image_id, metadata, data=None): return metadata - self.stubs.Set(local.LocalImageService, 'show', fake_show) - self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show) - self.stubs.Set(local.LocalImageService, 'update', fake_update) + self.stubs.Set(fake._FakeImageService, 'show', fake_show) + self.stubs.Set(fake._FakeImageService, 'show_by_name', fake_show) + self.stubs.Set(fake._FakeImageService, 'update', fake_update) result = modify_image_attribute(self.context, 'ami-00000001', 'launchPermission', 'add', user_group=['all']) @@ -285,7 +360,7 @@ class CloudTestCase(test.TestCase): def fake_delete(self, context, id): return None - self.stubs.Set(local.LocalImageService, 'delete', fake_delete) + self.stubs.Set(fake._FakeImageService, 'delete', fake_delete) # valid image result = deregister_image(self.context, 'ami-00000001') self.assertEqual(result['imageId'], 'ami-00000001') @@ -295,42 +370,41 @@ class CloudTestCase(test.TestCase): def fake_detail_empty(self, context): return [] - self.stubs.Set(local.LocalImageService, 'detail', fake_detail_empty) + self.stubs.Set(fake._FakeImageService, 'detail', fake_detail_empty) self.assertRaises(exception.ImageNotFound, deregister_image, self.context, 'ami-bad001') - def test_console_output(self): - instance_type = FLAGS.default_instance_type - max_count = 1 - kwargs = {'image_id': 'ami-1', - 'instance_type': instance_type, - 'max_count': max_count} + def _run_instance(self, **kwargs): rv = self.cloud.run_instances(self.context, **kwargs) - greenthread.sleep(0.3) instance_id = rv['instancesSet'][0]['instanceId'] + return instance_id + + def _run_instance_wait(self, **kwargs): + ec2_instance_id = self._run_instance(**kwargs) + self._wait_for_running(ec2_instance_id) + return ec2_instance_id + + def test_console_output(self): + instance_id = self._run_instance( + image_id='ami-1', + instance_type=FLAGS.default_instance_type, + max_count=1) output = self.cloud.get_console_output(context=self.context, instance_id=[instance_id]) self.assertEquals(b64decode(output['output']), 'FAKE CONSOLE?OUTPUT') # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. - greenthread.sleep(0.3) rv = self.cloud.terminate_instances(self.context, [instance_id]) - greenthread.sleep(0.3) def test_ajax_console(self): - kwargs = {'image_id': 'ami-1'} - rv = self.cloud.run_instances(self.context, **kwargs) - instance_id = rv['instancesSet'][0]['instanceId'] - greenthread.sleep(0.3) + instance_id = self._run_instance(image_id='ami-1') output = self.cloud.get_ajax_console(context=self.context, instance_id=[instance_id]) self.assertEquals(output['url'], '%s/?token=FAKETOKEN' % FLAGS.ajax_console_proxy_url) # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. - greenthread.sleep(0.3) rv = self.cloud.terminate_instances(self.context, [instance_id]) - greenthread.sleep(0.3) def test_key_generation(self): result = self._create_key('test') @@ -388,9 +462,73 @@ class CloudTestCase(test.TestCase): self._create_key('test') self.cloud.delete_key_pair(self.context, 'test') + def test_run_instances(self): + # stub out the rpc call + def stub_cast(*args, **kwargs): + pass + + self.stubs.Set(rpc, 'cast', stub_cast) + + kwargs = {'image_id': FLAGS.default_image, + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1} + run_instances = self.cloud.run_instances + result = run_instances(self.context, **kwargs) + instance = result['instancesSet'][0] + self.assertEqual(instance['imageId'], 'ami-00000001') + self.assertEqual(instance['displayName'], 'Server 1') + self.assertEqual(instance['instanceId'], 'i-00000001') + self.assertEqual(instance['instanceState']['name'], 'scheduling') + self.assertEqual(instance['instanceType'], 'm1.small') + + def test_run_instances_image_state_none(self): + kwargs = {'image_id': FLAGS.default_image, + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1} + run_instances = self.cloud.run_instances + + def fake_show_no_state(self, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, + 'type': 'machine'}} + + self.stubs.UnsetAll() + self.stubs.Set(fake._FakeImageService, 'show', fake_show_no_state) + self.assertRaises(exception.ApiError, run_instances, + self.context, **kwargs) + + def test_run_instances_image_state_invalid(self): + kwargs = {'image_id': FLAGS.default_image, + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1} + run_instances = self.cloud.run_instances + + def fake_show_decrypt(self, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, + 'type': 'machine', 'image_state': 'decrypting'}} + + self.stubs.UnsetAll() + self.stubs.Set(fake._FakeImageService, 'show', fake_show_decrypt) + self.assertRaises(exception.ApiError, run_instances, + self.context, **kwargs) + + def test_run_instances_image_status_active(self): + kwargs = {'image_id': FLAGS.default_image, + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1} + run_instances = self.cloud.run_instances + + def fake_show_stat_active(self, context, id): + return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, + 'type': 'machine'}, 'status': 'active'} + + self.stubs.Set(fake._FakeImageService, 'show', fake_show_stat_active) + + result = run_instances(self.context, **kwargs) + self.assertEqual(len(result['instancesSet']), 1) + def test_terminate_instances(self): inst1 = db.instance_create(self.context, {'reservation_id': 'a', - 'image_id': 1, + 'image_ref': 1, 'host': 'host1'}) terminate_instances = self.cloud.terminate_instances # valid instance_id @@ -412,7 +550,9 @@ class CloudTestCase(test.TestCase): def test_update_of_instance_wont_update_private_fields(self): inst = db.instance_create(self.context, {}) - self.cloud.update_instance(self.context, inst['id'], + ec2_id = ec2utils.id_to_ec2_id(inst['id']) + self.cloud.update_instance(self.context, ec2_id, + display_name='c00l 1m4g3', mac_address='DE:AD:BE:EF') inst = db.instance_get(self.context, inst['id']) self.assertEqual(None, inst['mac_address']) @@ -435,3 +575,299 @@ class CloudTestCase(test.TestCase): vol = db.volume_get(self.context, vol['id']) self.assertEqual(None, vol['mountpoint']) db.volume_destroy(self.context, vol['id']) + + def _restart_compute_service(self, periodic_interval=None): + """restart compute service. NOTE: fake driver forgets all instances.""" + self.compute.kill() + if periodic_interval: + self.compute = self.start_service( + 'compute', periodic_interval=periodic_interval) + else: + self.compute = self.start_service('compute') + + def _wait_for_state(self, ctxt, instance_id, predicate): + """Wait for an stopping instance to be a given state""" + id = ec2utils.ec2_id_to_id(instance_id) + while True: + info = self.cloud.compute_api.get(context=ctxt, instance_id=id) + LOG.debug(info) + if predicate(info): + break + greenthread.sleep(1) + + def _wait_for_running(self, instance_id): + def is_running(info): + return info['state_description'] == 'running' + self._wait_for_state(self.context, instance_id, is_running) + + def _wait_for_stopped(self, instance_id): + def is_stopped(info): + return info['state_description'] == 'stopped' + self._wait_for_state(self.context, instance_id, is_stopped) + + def _wait_for_terminate(self, instance_id): + def is_deleted(info): + return info['deleted'] + elevated = self.context.elevated(read_deleted=True) + self._wait_for_state(elevated, instance_id, is_deleted) + + def test_stop_start_instance(self): + """Makes sure stop/start instance works""" + # enforce periodic tasks run in short time to avoid wait for 60s. + self._restart_compute_service(periodic_interval=0.3) + + kwargs = {'image_id': 'ami-1', + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1, } + instance_id = self._run_instance_wait(**kwargs) + + # a running instance can't be started. It is just ignored. + result = self.cloud.start_instances(self.context, [instance_id]) + greenthread.sleep(0.3) + self.assertTrue(result) + + result = self.cloud.stop_instances(self.context, [instance_id]) + greenthread.sleep(0.3) + self.assertTrue(result) + self._wait_for_stopped(instance_id) + + result = self.cloud.start_instances(self.context, [instance_id]) + greenthread.sleep(0.3) + self.assertTrue(result) + self._wait_for_running(instance_id) + + result = self.cloud.stop_instances(self.context, [instance_id]) + greenthread.sleep(0.3) + self.assertTrue(result) + self._wait_for_stopped(instance_id) + + result = self.cloud.terminate_instances(self.context, [instance_id]) + greenthread.sleep(0.3) + self.assertTrue(result) + + self._restart_compute_service() + + def _volume_create(self): + kwargs = {'status': 'available', + 'host': self.volume.host, + 'size': 1, + 'attach_status': 'detached', } + return db.volume_create(self.context, kwargs) + + def _assert_volume_attached(self, vol, instance_id, mountpoint): + self.assertEqual(vol['instance_id'], instance_id) + self.assertEqual(vol['mountpoint'], mountpoint) + self.assertEqual(vol['status'], "in-use") + self.assertEqual(vol['attach_status'], "attached") + + def _assert_volume_detached(self, vol): + self.assertEqual(vol['instance_id'], None) + self.assertEqual(vol['mountpoint'], None) + self.assertEqual(vol['status'], "available") + self.assertEqual(vol['attach_status'], "detached") + + def test_stop_start_with_volume(self): + """Make sure run instance with block device mapping works""" + + # enforce periodic tasks run in short time to avoid wait for 60s. + self._restart_compute_service(periodic_interval=0.3) + + vol1 = self._volume_create() + vol2 = self._volume_create() + kwargs = {'image_id': 'ami-1', + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1, + 'block_device_mapping': [{'device_name': '/dev/vdb', + 'volume_id': vol1['id'], + 'delete_on_termination': False, }, + {'device_name': '/dev/vdc', + 'volume_id': vol2['id'], + 'delete_on_termination': True, }, + ]} + ec2_instance_id = self._run_instance_wait(**kwargs) + instance_id = ec2utils.ec2_id_to_id(ec2_instance_id) + + vols = db.volume_get_all_by_instance(self.context, instance_id) + self.assertEqual(len(vols), 2) + for vol in vols: + self.assertTrue(vol['id'] == vol1['id'] or vol['id'] == vol2['id']) + + vol = db.volume_get(self.context, vol1['id']) + self._assert_volume_attached(vol, instance_id, '/dev/vdb') + + vol = db.volume_get(self.context, vol2['id']) + self._assert_volume_attached(vol, instance_id, '/dev/vdc') + + result = self.cloud.stop_instances(self.context, [ec2_instance_id]) + self.assertTrue(result) + self._wait_for_stopped(ec2_instance_id) + + vol = db.volume_get(self.context, vol1['id']) + self._assert_volume_detached(vol) + vol = db.volume_get(self.context, vol2['id']) + self._assert_volume_detached(vol) + + self.cloud.start_instances(self.context, [ec2_instance_id]) + self._wait_for_running(ec2_instance_id) + vols = db.volume_get_all_by_instance(self.context, instance_id) + self.assertEqual(len(vols), 2) + for vol in vols: + self.assertTrue(vol['id'] == vol1['id'] or vol['id'] == vol2['id']) + self.assertTrue(vol['mountpoint'] == '/dev/vdb' or + vol['mountpoint'] == '/dev/vdc') + self.assertEqual(vol['instance_id'], instance_id) + self.assertEqual(vol['status'], "in-use") + self.assertEqual(vol['attach_status'], "attached") + + self.cloud.terminate_instances(self.context, [ec2_instance_id]) + greenthread.sleep(0.3) + + admin_ctxt = context.get_admin_context(read_deleted=False) + vol = db.volume_get(admin_ctxt, vol1['id']) + self.assertFalse(vol['deleted']) + db.volume_destroy(self.context, vol1['id']) + + greenthread.sleep(0.3) + admin_ctxt = context.get_admin_context(read_deleted=True) + vol = db.volume_get(admin_ctxt, vol2['id']) + self.assertTrue(vol['deleted']) + + self._restart_compute_service() + + def test_stop_with_attached_volume(self): + """Make sure attach info is reflected to block device mapping""" + # enforce periodic tasks run in short time to avoid wait for 60s. + self._restart_compute_service(periodic_interval=0.3) + + vol1 = self._volume_create() + vol2 = self._volume_create() + kwargs = {'image_id': 'ami-1', + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1, + 'block_device_mapping': [{'device_name': '/dev/vdb', + 'volume_id': vol1['id'], + 'delete_on_termination': True}]} + ec2_instance_id = self._run_instance_wait(**kwargs) + instance_id = ec2utils.ec2_id_to_id(ec2_instance_id) + + vols = db.volume_get_all_by_instance(self.context, instance_id) + self.assertEqual(len(vols), 1) + for vol in vols: + self.assertEqual(vol['id'], vol1['id']) + self._assert_volume_attached(vol, instance_id, '/dev/vdb') + + vol = db.volume_get(self.context, vol2['id']) + self._assert_volume_detached(vol) + + self.cloud.compute_api.attach_volume(self.context, + instance_id=instance_id, + volume_id=vol2['id'], + device='/dev/vdc') + greenthread.sleep(0.3) + vol = db.volume_get(self.context, vol2['id']) + self._assert_volume_attached(vol, instance_id, '/dev/vdc') + + self.cloud.compute_api.detach_volume(self.context, + volume_id=vol1['id']) + greenthread.sleep(0.3) + vol = db.volume_get(self.context, vol1['id']) + self._assert_volume_detached(vol) + + result = self.cloud.stop_instances(self.context, [ec2_instance_id]) + self.assertTrue(result) + self._wait_for_stopped(ec2_instance_id) + + for vol_id in (vol1['id'], vol2['id']): + vol = db.volume_get(self.context, vol_id) + self._assert_volume_detached(vol) + + self.cloud.start_instances(self.context, [ec2_instance_id]) + self._wait_for_running(ec2_instance_id) + vols = db.volume_get_all_by_instance(self.context, instance_id) + self.assertEqual(len(vols), 1) + for vol in vols: + self.assertEqual(vol['id'], vol2['id']) + self._assert_volume_attached(vol, instance_id, '/dev/vdc') + + vol = db.volume_get(self.context, vol1['id']) + self._assert_volume_detached(vol) + + self.cloud.terminate_instances(self.context, [ec2_instance_id]) + greenthread.sleep(0.3) + + for vol_id in (vol1['id'], vol2['id']): + vol = db.volume_get(self.context, vol_id) + self.assertEqual(vol['id'], vol_id) + self._assert_volume_detached(vol) + db.volume_destroy(self.context, vol_id) + + self._restart_compute_service() + + def _create_snapshot(self, ec2_volume_id): + result = self.cloud.create_snapshot(self.context, + volume_id=ec2_volume_id) + greenthread.sleep(0.3) + return result['snapshotId'] + + def test_run_with_snapshot(self): + """Makes sure run/stop/start instance with snapshot works.""" + vol = self._volume_create() + ec2_volume_id = ec2utils.id_to_ec2_id(vol['id'], 'vol-%08x') + + ec2_snapshot1_id = self._create_snapshot(ec2_volume_id) + snapshot1_id = ec2utils.ec2_id_to_id(ec2_snapshot1_id) + ec2_snapshot2_id = self._create_snapshot(ec2_volume_id) + snapshot2_id = ec2utils.ec2_id_to_id(ec2_snapshot2_id) + + kwargs = {'image_id': 'ami-1', + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1, + 'block_device_mapping': [{'device_name': '/dev/vdb', + 'snapshot_id': snapshot1_id, + 'delete_on_termination': False, }, + {'device_name': '/dev/vdc', + 'snapshot_id': snapshot2_id, + 'delete_on_termination': True}]} + ec2_instance_id = self._run_instance_wait(**kwargs) + instance_id = ec2utils.ec2_id_to_id(ec2_instance_id) + + vols = db.volume_get_all_by_instance(self.context, instance_id) + self.assertEqual(len(vols), 2) + vol1_id = None + vol2_id = None + for vol in vols: + snapshot_id = vol['snapshot_id'] + if snapshot_id == snapshot1_id: + vol1_id = vol['id'] + mountpoint = '/dev/vdb' + elif snapshot_id == snapshot2_id: + vol2_id = vol['id'] + mountpoint = '/dev/vdc' + else: + self.fail() + + self._assert_volume_attached(vol, instance_id, mountpoint) + + self.assertTrue(vol1_id) + self.assertTrue(vol2_id) + + self.cloud.terminate_instances(self.context, [ec2_instance_id]) + greenthread.sleep(0.3) + self._wait_for_terminate(ec2_instance_id) + + greenthread.sleep(0.3) + admin_ctxt = context.get_admin_context(read_deleted=False) + vol = db.volume_get(admin_ctxt, vol1_id) + self._assert_volume_detached(vol) + self.assertFalse(vol['deleted']) + db.volume_destroy(self.context, vol1_id) + + greenthread.sleep(0.3) + admin_ctxt = context.get_admin_context(read_deleted=True) + vol = db.volume_get(admin_ctxt, vol2_id) + self.assertTrue(vol['deleted']) + + for snapshot_id in (ec2_snapshot1_id, ec2_snapshot2_id): + self.cloud.delete_snapshot(self.context, snapshot_id) + greenthread.sleep(0.3) + db.volume_destroy(self.context, vol['id']) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index 9170837b6..439508b27 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -19,25 +19,24 @@ Tests For Compute """ -import datetime import mox import stubout +from nova.auth import manager from nova import compute +from nova.compute import instance_types +from nova.compute import manager as compute_manager +from nova.compute import power_state from nova import context from nova import db +from nova.db.sqlalchemy import models from nova import exception from nova import flags +import nova.image.fake from nova import log as logging from nova import rpc from nova import test from nova import utils -from nova.auth import manager -from nova.compute import instance_types -from nova.compute import manager as compute_manager -from nova.compute import power_state -from nova.db.sqlalchemy import models -from nova.image import local LOG = logging.getLogger('nova.tests.compute') FLAGS = flags.FLAGS @@ -74,7 +73,7 @@ class ComputeTestCase(test.TestCase): def fake_show(meh, context, id): return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}} - self.stubs.Set(local.LocalImageService, 'show', fake_show) + self.stubs.Set(nova.image.fake._FakeImageService, 'show', fake_show) def tearDown(self): self.manager.delete_user(self.user) @@ -84,7 +83,7 @@ class ComputeTestCase(test.TestCase): def _create_instance(self, params={}): """Create a test instance""" inst = {} - inst['image_id'] = 1 + inst['image_ref'] = 1 inst['reservation_id'] = 'r-fakeres' inst['launch_time'] = '10' inst['user_id'] = self.user.id @@ -150,7 +149,7 @@ class ComputeTestCase(test.TestCase): ref = self.compute_api.create( self.context, instance_type=instance_types.get_default_instance_type(), - image_id=None, + image_href=None, security_group=['testgroup']) try: self.assertEqual(len(db.security_group_get_by_instance( @@ -168,7 +167,7 @@ class ComputeTestCase(test.TestCase): ref = self.compute_api.create( self.context, instance_type=instance_types.get_default_instance_type(), - image_id=None, + image_href=None, security_group=['testgroup']) try: db.instance_destroy(self.context, ref[0]['id']) @@ -184,7 +183,7 @@ class ComputeTestCase(test.TestCase): ref = self.compute_api.create( self.context, instance_type=instance_types.get_default_instance_type(), - image_id=None, + image_href=None, security_group=['testgroup']) try: @@ -217,18 +216,33 @@ class ComputeTestCase(test.TestCase): instance_ref = db.instance_get(self.context, instance_id) self.assertEqual(instance_ref['launched_at'], None) self.assertEqual(instance_ref['deleted_at'], None) - launch = datetime.datetime.utcnow() + launch = utils.utcnow() self.compute.run_instance(self.context, instance_id) instance_ref = db.instance_get(self.context, instance_id) self.assert_(instance_ref['launched_at'] > launch) self.assertEqual(instance_ref['deleted_at'], None) - terminate = datetime.datetime.utcnow() + terminate = utils.utcnow() self.compute.terminate_instance(self.context, instance_id) self.context = self.context.elevated(True) instance_ref = db.instance_get(self.context, instance_id) self.assert_(instance_ref['launched_at'] < terminate) self.assert_(instance_ref['deleted_at'] > terminate) + def test_stop(self): + """Ensure instance can be stopped""" + instance_id = self._create_instance() + self.compute.run_instance(self.context, instance_id) + self.compute.stop_instance(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) + + def test_start(self): + """Ensure instance can be started""" + instance_id = self._create_instance() + self.compute.run_instance(self.context, instance_id) + self.compute.stop_instance(self.context, instance_id) + self.compute.start_instance(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) + def test_pause(self): """Ensure instance can be paused""" instance_id = self._create_instance() @@ -267,6 +281,14 @@ class ComputeTestCase(test.TestCase): "File Contents") self.compute.terminate_instance(self.context, instance_id) + def test_agent_update(self): + """Ensure instance can have its agent updated""" + instance_id = self._create_instance() + self.compute.run_instance(self.context, instance_id) + self.compute.agent_update(self.context, instance_id, + 'http://127.0.0.1/agent', '00112233445566778899aabbccddeeff') + self.compute.terminate_instance(self.context, instance_id) + def test_snapshot(self): """Ensure instance can be snapshotted""" instance_id = self._create_instance() diff --git a/nova/tests/test_console.py b/nova/tests/test_console.py index 1a9a867ee..831e7670f 100644 --- a/nova/tests/test_console.py +++ b/nova/tests/test_console.py @@ -20,8 +20,6 @@ Tests For Console proxy. """ -import datetime - from nova import context from nova import db from nova import exception diff --git a/nova/tests/test_crypto.py b/nova/tests/test_crypto.py index 945d78794..6c25b396e 100644 --- a/nova/tests/test_crypto.py +++ b/nova/tests/test_crypto.py @@ -16,7 +16,11 @@ Tests for Crypto module. """ +import mox +import stubout + from nova import crypto +from nova import db from nova import test @@ -46,3 +50,82 @@ class SymmetricKeyTestCase(test.TestCase): plain = decrypt(cipher_text) self.assertEquals(plain_text, plain) + + +class RevokeCertsTest(test.TestCase): + + def setUp(self): + super(RevokeCertsTest, self).setUp() + self.stubs = stubout.StubOutForTesting() + + def tearDown(self): + self.stubs.UnsetAll() + super(RevokeCertsTest, self).tearDown() + + def test_revoke_certs_by_user_and_project(self): + user_id = 'test_user' + project_id = 2 + file_name = 'test_file' + + def mock_certificate_get_all_by_user_and_project(context, + user_id, + project_id): + + return [{"user_id": user_id, "project_id": project_id, + "file_name": file_name}] + + self.stubs.Set(db, 'certificate_get_all_by_user_and_project', + mock_certificate_get_all_by_user_and_project) + + self.mox.StubOutWithMock(crypto, 'revoke_cert') + crypto.revoke_cert(project_id, file_name) + + self.mox.ReplayAll() + + crypto.revoke_certs_by_user_and_project(user_id, project_id) + + self.mox.VerifyAll() + + def test_revoke_certs_by_user(self): + user_id = 'test_user' + project_id = 2 + file_name = 'test_file' + + def mock_certificate_get_all_by_user(context, user_id): + + return [{"user_id": user_id, "project_id": project_id, + "file_name": file_name}] + + self.stubs.Set(db, 'certificate_get_all_by_user', + mock_certificate_get_all_by_user) + + self.mox.StubOutWithMock(crypto, 'revoke_cert') + crypto.revoke_cert(project_id, mox.IgnoreArg()) + + self.mox.ReplayAll() + + crypto.revoke_certs_by_user(user_id) + + self.mox.VerifyAll() + + def test_revoke_certs_by_project(self): + user_id = 'test_user' + project_id = 2 + file_name = 'test_file' + + def mock_certificate_get_all_by_project(context, project_id): + + return [{"user_id": user_id, "project_id": project_id, + "file_name": file_name}] + + self.stubs.Set(db, 'certificate_get_all_by_project', + mock_certificate_get_all_by_project) + + self.mox.StubOutWithMock(crypto, 'revoke_cert') + crypto.revoke_cert(project_id, mox.IgnoreArg()) + + self.mox.ReplayAll() + + crypto.revoke_certs_by_project(project_id) + + self.mox.VerifyAll() diff --git a/nova/tests/test_flags.py b/nova/tests/test_flags.py index 707300fcf..05319d91f 100644 --- a/nova/tests/test_flags.py +++ b/nova/tests/test_flags.py @@ -91,6 +91,20 @@ class FlagsTestCase(test.TestCase): self.assert_('runtime_answer' in self.global_FLAGS) self.assertEqual(self.global_FLAGS.runtime_answer, 60) + def test_long_vs_short_flags(self): + flags.DEFINE_string('duplicate_answer_long', 'val', 'desc', + flag_values=self.global_FLAGS) + argv = ['flags_test', '--duplicate_answer=60', 'extra_arg'] + args = self.global_FLAGS(argv) + + self.assert_('duplicate_answer' not in self.global_FLAGS) + self.assert_(self.global_FLAGS.duplicate_answer_long, 60) + + flags.DEFINE_integer('duplicate_answer', 60, 'desc', + flag_values=self.global_FLAGS) + self.assertEqual(self.global_FLAGS.duplicate_answer, 60) + self.assertEqual(self.global_FLAGS.duplicate_answer_long, 'val') + def test_flag_leak_left(self): self.assertEqual(FLAGS.flags_unittest, 'foo') FLAGS.flags_unittest = 'bar' diff --git a/nova/tests/test_flat_network.py b/nova/tests/test_flat_network.py index dcc617e25..8544019c0 100644 --- a/nova/tests/test_flat_network.py +++ b/nova/tests/test_flat_network.py @@ -18,7 +18,7 @@ """ Unit Tests for flat network code """ -import IPy +import netaddr import os import unittest @@ -45,8 +45,8 @@ class FlatNetworkTestCase(base.NetworkTestCase): self.context._project = self.projects[0] self.context.project_id = self.projects[0].id - pubnet = IPy.IP(flags.FLAGS.floating_range) - address = str(pubnet[0]) + pubnet = netaddr.IPRange(flags.FLAGS.floating_range) + address = str(list(pubnet)[0]) try: db.floating_ip_get_by_address(context.get_admin_context(), address) except exception.NotFound: diff --git a/nova/tests/test_host_filter.py b/nova/tests/test_host_filter.py index c029d41e6..3361c7b73 100644 --- a/nova/tests/test_host_filter.py +++ b/nova/tests/test_host_filter.py @@ -13,7 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. """ -Tests For Scheduler Host Filter Drivers. +Tests For Scheduler Host Filters. """ import json @@ -31,7 +31,7 @@ class FakeZoneManager: class HostFilterTestCase(test.TestCase): - """Test case for host filter drivers.""" + """Test case for host filters.""" def _host_caps(self, multiplier): # Returns host capabilities in the following way: @@ -57,8 +57,8 @@ class HostFilterTestCase(test.TestCase): 'host_name-label': 'xs-%s' % multiplier} def setUp(self): - self.old_flag = FLAGS.default_host_filter_driver - FLAGS.default_host_filter_driver = \ + self.old_flag = FLAGS.default_host_filter + FLAGS.default_host_filter = \ 'nova.scheduler.host_filter.AllHostsFilter' self.instance_type = dict(name='tiny', memory_mb=50, @@ -76,51 +76,52 @@ class HostFilterTestCase(test.TestCase): self.zone_manager.service_states = states def tearDown(self): - FLAGS.default_host_filter_driver = self.old_flag + FLAGS.default_host_filter = self.old_flag - def test_choose_driver(self): - # Test default driver ... - driver = host_filter.choose_driver() - self.assertEquals(driver._full_name(), + def test_choose_filter(self): + # Test default filter ... + hf = host_filter.choose_host_filter() + self.assertEquals(hf._full_name(), 'nova.scheduler.host_filter.AllHostsFilter') - # Test valid driver ... - driver = host_filter.choose_driver( - 'nova.scheduler.host_filter.FlavorFilter') - self.assertEquals(driver._full_name(), - 'nova.scheduler.host_filter.FlavorFilter') - # Test invalid driver ... + # Test valid filter ... + hf = host_filter.choose_host_filter( + 'nova.scheduler.host_filter.InstanceTypeFilter') + self.assertEquals(hf._full_name(), + 'nova.scheduler.host_filter.InstanceTypeFilter') + # Test invalid filter ... try: - host_filter.choose_driver('does not exist') - self.fail("Should not find driver") - except exception.SchedulerHostFilterDriverNotFound: + host_filter.choose_host_filter('does not exist') + self.fail("Should not find host filter.") + except exception.SchedulerHostFilterNotFound: pass - def test_all_host_driver(self): - driver = host_filter.AllHostsFilter() - cooked = driver.instance_type_to_filter(self.instance_type) - hosts = driver.filter_hosts(self.zone_manager, cooked) + def test_all_host_filter(self): + hf = host_filter.AllHostsFilter() + cooked = hf.instance_type_to_filter(self.instance_type) + hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(10, len(hosts)) for host, capabilities in hosts: self.assertTrue(host.startswith('host')) - def test_flavor_driver(self): - driver = host_filter.FlavorFilter() + def test_instance_type_filter(self): + hf = host_filter.InstanceTypeFilter() # filter all hosts that can support 50 ram and 500 disk - name, cooked = driver.instance_type_to_filter(self.instance_type) - self.assertEquals('nova.scheduler.host_filter.FlavorFilter', name) - hosts = driver.filter_hosts(self.zone_manager, cooked) + name, cooked = hf.instance_type_to_filter(self.instance_type) + self.assertEquals('nova.scheduler.host_filter.InstanceTypeFilter', + name) + hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() self.assertEquals('host05', just_hosts[0]) self.assertEquals('host10', just_hosts[5]) - def test_json_driver(self): - driver = host_filter.JsonFilter() + def test_json_filter(self): + hf = host_filter.JsonFilter() # filter all hosts that can support 50 ram and 500 disk - name, cooked = driver.instance_type_to_filter(self.instance_type) + name, cooked = hf.instance_type_to_filter(self.instance_type) self.assertEquals('nova.scheduler.host_filter.JsonFilter', name) - hosts = driver.filter_hosts(self.zone_manager, cooked) + hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(6, len(hosts)) just_hosts = [host for host, caps in hosts] just_hosts.sort() @@ -132,15 +133,16 @@ class HostFilterTestCase(test.TestCase): raw = ['or', ['and', ['<', '$compute.host_memory_free', 30], - ['<', '$compute.disk_available', 300] + ['<', '$compute.disk_available', 300], ], ['and', ['>', '$compute.host_memory_free', 70], - ['>', '$compute.disk_available', 700] - ] + ['>', '$compute.disk_available', 700], + ], ] + cooked = json.dumps(raw) - hosts = driver.filter_hosts(self.zone_manager, cooked) + hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(5, len(hosts)) just_hosts = [host for host, caps in hosts] @@ -152,7 +154,7 @@ class HostFilterTestCase(test.TestCase): ['=', '$compute.host_memory_free', 30], ] cooked = json.dumps(raw) - hosts = driver.filter_hosts(self.zone_manager, cooked) + hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(9, len(hosts)) just_hosts = [host for host, caps in hosts] @@ -162,7 +164,7 @@ class HostFilterTestCase(test.TestCase): raw = ['in', '$compute.host_memory_free', 20, 40, 60, 80, 100] cooked = json.dumps(raw) - hosts = driver.filter_hosts(self.zone_manager, cooked) + hosts = hf.filter_hosts(self.zone_manager, cooked) self.assertEquals(5, len(hosts)) just_hosts = [host for host, caps in hosts] @@ -174,35 +176,30 @@ class HostFilterTestCase(test.TestCase): raw = ['unknown command', ] cooked = json.dumps(raw) try: - driver.filter_hosts(self.zone_manager, cooked) + hf.filter_hosts(self.zone_manager, cooked) self.fail("Should give KeyError") except KeyError, e: pass - self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps([]))) - self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps({}))) - self.assertTrue(driver.filter_hosts(self.zone_manager, json.dumps( - ['not', True, False, True, False] - ))) + self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps([]))) + self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps({}))) + self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps( + ['not', True, False, True, False]))) try: - driver.filter_hosts(self.zone_manager, json.dumps( - 'not', True, False, True, False - )) + hf.filter_hosts(self.zone_manager, json.dumps( + 'not', True, False, True, False)) self.fail("Should give KeyError") except KeyError, e: pass - self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( - ['=', '$foo', 100] - ))) - self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( - ['=', '$.....', 100] - ))) - self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( - ['>', ['and', ['or', ['not', ['<', ['>=', ['<=', ['in', ]]]]]]]] - ))) - - self.assertFalse(driver.filter_hosts(self.zone_manager, json.dumps( - ['=', {}, ['>', '$missing....foo']] - ))) + self.assertFalse(hf.filter_hosts(self.zone_manager, + json.dumps(['=', '$foo', 100]))) + self.assertFalse(hf.filter_hosts(self.zone_manager, + json.dumps(['=', '$.....', 100]))) + self.assertFalse(hf.filter_hosts(self.zone_manager, + json.dumps( + ['>', ['and', ['or', ['not', ['<', ['>=', ['<=', ['in', ]]]]]]]]))) + + self.assertFalse(hf.filter_hosts(self.zone_manager, + json.dumps(['=', {}, ['>', '$missing....foo']]))) diff --git a/nova/tests/test_virt.py b/nova/tests/test_libvirt.py index 1bec9caca..d12e21063 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_libvirt.py @@ -14,10 +14,12 @@ # License for the specific language governing permissions and limitations # under the License. +import copy import eventlet import mox import os import re +import shutil import sys from xml.etree.ElementTree import fromstring as xml_to_tree @@ -32,7 +34,8 @@ from nova import utils from nova.api.ec2 import cloud from nova.auth import manager from nova.compute import power_state -from nova.virt import libvirt_conn +from nova.virt.libvirt import connection +from nova.virt.libvirt import firewall libvirt = None FLAGS = flags.FLAGS @@ -83,7 +86,7 @@ class CacheConcurrencyTestCase(test.TestCase): def test_same_fname_concurrency(self): """Ensures that the same fname cache runs at a sequentially""" - conn = libvirt_conn.LibvirtConnection + conn = connection.LibvirtConnection wait1 = eventlet.event.Event() done1 = eventlet.event.Event() eventlet.spawn(conn._cache_image, _concurrency, @@ -104,7 +107,7 @@ class CacheConcurrencyTestCase(test.TestCase): def test_different_fname_concurrency(self): """Ensures that two different fname caches are concurrent""" - conn = libvirt_conn.LibvirtConnection + conn = connection.LibvirtConnection wait1 = eventlet.event.Event() done1 = eventlet.event.Event() eventlet.spawn(conn._cache_image, _concurrency, @@ -123,9 +126,10 @@ class CacheConcurrencyTestCase(test.TestCase): class LibvirtConnTestCase(test.TestCase): + def setUp(self): super(LibvirtConnTestCase, self).setUp() - libvirt_conn._late_load_cheetah() + connection._late_load_cheetah() self.flags(fake_call=True) self.manager = manager.AuthManager() @@ -159,6 +163,7 @@ class LibvirtConnTestCase(test.TestCase): 'vcpus': 2, 'project_id': 'fake', 'bridge': 'br101', + 'image_ref': '123456', 'instance_type_id': '5'} # m1.small def lazy_load_library_exists(self): @@ -171,8 +176,8 @@ class LibvirtConnTestCase(test.TestCase): return False global libvirt libvirt = __import__('libvirt') - libvirt_conn.libvirt = __import__('libvirt') - libvirt_conn.libxml2 = __import__('libxml2') + connection.libvirt = __import__('libvirt') + connection.libxml2 = __import__('libxml2') return True def create_fake_libvirt_mock(self, **kwargs): @@ -182,7 +187,7 @@ class LibvirtConnTestCase(test.TestCase): class FakeLibvirtConnection(object): pass - # A fake libvirt_conn.IptablesFirewallDriver + # A fake connection.IptablesFirewallDriver class FakeIptablesFirewallDriver(object): def __init__(self, **kwargs): @@ -198,11 +203,34 @@ class LibvirtConnTestCase(test.TestCase): for key, val in kwargs.items(): fake.__setattr__(key, val) - # Inevitable mocks for libvirt_conn.LibvirtConnection - self.mox.StubOutWithMock(libvirt_conn.utils, 'import_class') - libvirt_conn.utils.import_class(mox.IgnoreArg()).AndReturn(fakeip) - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, '_conn') - libvirt_conn.LibvirtConnection._conn = fake + # Inevitable mocks for connection.LibvirtConnection + self.mox.StubOutWithMock(connection.utils, 'import_class') + connection.utils.import_class(mox.IgnoreArg()).AndReturn(fakeip) + self.mox.StubOutWithMock(connection.LibvirtConnection, '_conn') + connection.LibvirtConnection._conn = fake + + def fake_lookup(self, instance_name): + + class FakeVirtDomain(object): + + def snapshotCreateXML(self, *args): + return None + + def XMLDesc(self, *args): + return """ + <domain type='kvm'> + <devices> + <disk type='file'> + <source file='filename'/> + </disk> + </devices> + </domain> + """ + + return FakeVirtDomain() + + def fake_execute(self, *args): + open(args[-1], "a").close() def create_service(self, **kwargs): service_ref = {'host': kwargs.get('host', 'dummy'), @@ -214,7 +242,7 @@ class LibvirtConnTestCase(test.TestCase): return db.service_create(context.get_admin_context(), service_ref) def test_preparing_xml_info(self): - conn = libvirt_conn.LibvirtConnection(True) + conn = connection.LibvirtConnection(True) instance_ref = db.instance_create(self.context, self.test_instance) result = conn._prepare_xml_info(instance_ref, False) @@ -229,7 +257,7 @@ class LibvirtConnTestCase(test.TestCase): self.assertTrue(len(result['nics']) == 2) def test_get_nic_for_xml_v4(self): - conn = libvirt_conn.LibvirtConnection(True) + conn = connection.LibvirtConnection(True) network, mapping = _create_network_info()[0] self.flags(use_ipv6=False) params = conn._get_nic_for_xml(network, mapping)['extra_params'] @@ -237,7 +265,7 @@ class LibvirtConnTestCase(test.TestCase): self.assertTrue(params.find('PROJMASKV6') == -1) def test_get_nic_for_xml_v6(self): - conn = libvirt_conn.LibvirtConnection(True) + conn = connection.LibvirtConnection(True) network, mapping = _create_network_info()[0] self.flags(use_ipv6=True) params = conn._get_nic_for_xml(network, mapping)['extra_params'] @@ -279,10 +307,85 @@ class LibvirtConnTestCase(test.TestCase): instance_data = dict(self.test_instance) self._check_xml_and_container(instance_data) + def test_snapshot(self): + if not self.lazy_load_library_exists(): + return + + FLAGS.image_service = 'nova.image.fake.FakeImageService' + + # Start test + image_service = utils.import_object(FLAGS.image_service) + + # Assuming that base image already exists in image_service + instance_ref = db.instance_create(self.context, self.test_instance) + properties = {'instance_id': instance_ref['id'], + 'user_id': str(self.context.user_id)} + snapshot_name = 'test-snap' + sent_meta = {'name': snapshot_name, 'is_public': False, + 'status': 'creating', 'properties': properties} + # Create new image. It will be updated in snapshot method + # To work with it from snapshot, the single image_service is needed + recv_meta = image_service.create(context, sent_meta) + + self.mox.StubOutWithMock(connection.LibvirtConnection, '_conn') + connection.LibvirtConnection._conn.lookupByName = self.fake_lookup + self.mox.StubOutWithMock(connection.utils, 'execute') + connection.utils.execute = self.fake_execute + + self.mox.ReplayAll() + + conn = connection.LibvirtConnection(False) + conn.snapshot(instance_ref, recv_meta['id']) + + snapshot = image_service.show(context, recv_meta['id']) + self.assertEquals(snapshot['properties']['image_state'], 'available') + self.assertEquals(snapshot['status'], 'active') + self.assertEquals(snapshot['name'], snapshot_name) + + def test_snapshot_no_image_architecture(self): + if not self.lazy_load_library_exists(): + return + + FLAGS.image_service = 'nova.image.fake.FakeImageService' + + # Start test + image_service = utils.import_object(FLAGS.image_service) + + # Assign image_ref = 2 from nova/images/fakes for testing different + # base image + test_instance = copy.deepcopy(self.test_instance) + test_instance["image_ref"] = "2" + + # Assuming that base image already exists in image_service + instance_ref = db.instance_create(self.context, test_instance) + properties = {'instance_id': instance_ref['id'], + 'user_id': str(self.context.user_id)} + snapshot_name = 'test-snap' + sent_meta = {'name': snapshot_name, 'is_public': False, + 'status': 'creating', 'properties': properties} + # Create new image. It will be updated in snapshot method + # To work with it from snapshot, the single image_service is needed + recv_meta = image_service.create(context, sent_meta) + + self.mox.StubOutWithMock(connection.LibvirtConnection, '_conn') + connection.LibvirtConnection._conn.lookupByName = self.fake_lookup + self.mox.StubOutWithMock(connection.utils, 'execute') + connection.utils.execute = self.fake_execute + + self.mox.ReplayAll() + + conn = connection.LibvirtConnection(False) + conn.snapshot(instance_ref, recv_meta['id']) + + snapshot = image_service.show(context, recv_meta['id']) + self.assertEquals(snapshot['properties']['image_state'], 'available') + self.assertEquals(snapshot['status'], 'active') + self.assertEquals(snapshot['name'], snapshot_name) + def test_multi_nic(self): instance_data = dict(self.test_instance) network_info = _create_network_info(2) - conn = libvirt_conn.LibvirtConnection(True) + conn = connection.LibvirtConnection(True) instance_ref = db.instance_create(self.context, instance_data) xml = conn.to_xml(instance_ref, False, network_info) tree = xml_to_tree(xml) @@ -313,7 +416,7 @@ class LibvirtConnTestCase(test.TestCase): 'instance_id': instance_ref['id']}) self.flags(libvirt_type='lxc') - conn = libvirt_conn.LibvirtConnection(True) + conn = connection.LibvirtConnection(True) uri = conn.get_uri() self.assertEquals(uri, 'lxc:///') @@ -419,7 +522,7 @@ class LibvirtConnTestCase(test.TestCase): for (libvirt_type, (expected_uri, checks)) in type_uri_map.iteritems(): FLAGS.libvirt_type = libvirt_type - conn = libvirt_conn.LibvirtConnection(True) + conn = connection.LibvirtConnection(True) uri = conn.get_uri() self.assertEquals(uri, expected_uri) @@ -446,7 +549,7 @@ class LibvirtConnTestCase(test.TestCase): FLAGS.libvirt_uri = testuri for (libvirt_type, (expected_uri, checks)) in type_uri_map.iteritems(): FLAGS.libvirt_type = libvirt_type - conn = libvirt_conn.LibvirtConnection(True) + conn = connection.LibvirtConnection(True) uri = conn.get_uri() self.assertEquals(uri, testuri) db.instance_destroy(user_context, instance_ref['id']) @@ -470,13 +573,13 @@ class LibvirtConnTestCase(test.TestCase): self.create_fake_libvirt_mock(getVersion=getVersion, getType=getType, listDomainsID=listDomainsID) - self.mox.StubOutWithMock(libvirt_conn.LibvirtConnection, + self.mox.StubOutWithMock(connection.LibvirtConnection, 'get_cpu_info') - libvirt_conn.LibvirtConnection.get_cpu_info().AndReturn('cpuinfo') + connection.LibvirtConnection.get_cpu_info().AndReturn('cpuinfo') # Start test self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) + conn = connection.LibvirtConnection(False) conn.update_available_resource(self.context, 'dummy') service_ref = db.service_get(self.context, service_ref['id']) compute_node = service_ref['compute_node'][0] @@ -510,7 +613,7 @@ class LibvirtConnTestCase(test.TestCase): self.create_fake_libvirt_mock() self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) + conn = connection.LibvirtConnection(False) self.assertRaises(exception.ComputeServiceUnavailable, conn.update_available_resource, self.context, 'dummy') @@ -545,7 +648,7 @@ class LibvirtConnTestCase(test.TestCase): # Start test self.mox.ReplayAll() try: - conn = libvirt_conn.LibvirtConnection(False) + conn = connection.LibvirtConnection(False) conn.firewall_driver.setattr('setup_basic_filtering', fake_none) conn.firewall_driver.setattr('prepare_instance_filter', fake_none) conn.firewall_driver.setattr('instance_filter_exists', fake_none) @@ -594,7 +697,7 @@ class LibvirtConnTestCase(test.TestCase): # Start test self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) + conn = connection.LibvirtConnection(False) self.assertRaises(libvirt.libvirtError, conn._live_migration, self.context, instance_ref, 'dest', '', @@ -623,7 +726,7 @@ class LibvirtConnTestCase(test.TestCase): # Start test self.mox.ReplayAll() - conn = libvirt_conn.LibvirtConnection(False) + conn = connection.LibvirtConnection(False) conn.firewall_driver.setattr('setup_basic_filtering', fake_none) conn.firewall_driver.setattr('prepare_instance_filter', fake_none) @@ -644,10 +747,12 @@ class LibvirtConnTestCase(test.TestCase): except Exception, e: count = (0 <= str(e.message).find('Unexpected method call')) + shutil.rmtree(os.path.join(FLAGS.instances_path, instance.name)) + self.assertTrue(count) def test_get_host_ip_addr(self): - conn = libvirt_conn.LibvirtConnection(False) + conn = connection.LibvirtConnection(False) ip = conn.get_host_ip_addr() self.assertEquals(ip, FLAGS.my_ip) @@ -657,6 +762,31 @@ class LibvirtConnTestCase(test.TestCase): super(LibvirtConnTestCase, self).tearDown() +class NWFilterFakes: + def __init__(self): + self.filters = {} + + def nwfilterLookupByName(self, name): + if name in self.filters: + return self.filters[name] + raise libvirt.libvirtError('Filter Not Found') + + def filterDefineXMLMock(self, xml): + class FakeNWFilterInternal: + def __init__(self, parent, name): + self.name = name + self.parent = parent + + def undefine(self): + del self.parent.filters[self.name] + pass + tree = xml_to_tree(xml) + name = tree.get('name') + if name not in self.filters: + self.filters[name] = FakeNWFilterInternal(self, name) + return True + + class IptablesFirewallTestCase(test.TestCase): def setUp(self): super(IptablesFirewallTestCase, self).setUp() @@ -669,11 +799,27 @@ class IptablesFirewallTestCase(test.TestCase): self.network = utils.import_object(FLAGS.network_manager) class FakeLibvirtConnection(object): - pass + def nwfilterDefineXML(*args, **kwargs): + """setup_basic_rules in nwfilter calls this.""" + pass self.fake_libvirt_connection = FakeLibvirtConnection() - self.fw = libvirt_conn.IptablesFirewallDriver( + self.fw = firewall.IptablesFirewallDriver( get_connection=lambda: self.fake_libvirt_connection) + def lazy_load_library_exists(self): + """check if libvirt is available.""" + # try to connect libvirt. if fail, skip test. + try: + import libvirt + import libxml2 + except ImportError: + return False + global libvirt + libvirt = __import__('libvirt') + connection.libvirt = __import__('libvirt') + connection.libxml2 = __import__('libxml2') + return True + def tearDown(self): self.manager.delete_project(self.project) self.manager.delete_user(self.user) @@ -879,6 +1025,103 @@ class IptablesFirewallTestCase(test.TestCase): self.mox.ReplayAll() self.fw.do_refresh_security_group_rules("fake") + def test_unfilter_instance_undefines_nwfilter(self): + # Skip if non-libvirt environment + if not self.lazy_load_library_exists(): + return + + admin_ctxt = context.get_admin_context() + + fakefilter = NWFilterFakes() + self.fw.nwfilter._conn.nwfilterDefineXML =\ + fakefilter.filterDefineXMLMock + self.fw.nwfilter._conn.nwfilterLookupByName =\ + fakefilter.nwfilterLookupByName + instance_ref = self._create_instance_ref() + inst_id = instance_ref['id'] + instance = db.instance_get(self.context, inst_id) + + ip = '10.11.12.13' + network_ref = db.project_get_network(self.context, 'fake') + fixed_ip = {'address': ip, 'network_id': network_ref['id']} + db.fixed_ip_create(admin_ctxt, fixed_ip) + db.fixed_ip_update(admin_ctxt, ip, {'allocated': True, + 'instance_id': inst_id}) + self.fw.setup_basic_filtering(instance) + self.fw.prepare_instance_filter(instance) + self.fw.apply_instance_filter(instance) + original_filter_count = len(fakefilter.filters) + self.fw.unfilter_instance(instance) + + # should undefine just the instance filter + self.assertEqual(original_filter_count - len(fakefilter.filters), 1) + + db.instance_destroy(admin_ctxt, instance_ref['id']) + + def test_provider_firewall_rules(self): + # setup basic instance data + instance_ref = self._create_instance_ref() + nw_info = _create_network_info(1) + ip = '10.11.12.13' + network_ref = db.project_get_network(self.context, 'fake') + admin_ctxt = context.get_admin_context() + fixed_ip = {'address': ip, 'network_id': network_ref['id']} + db.fixed_ip_create(admin_ctxt, fixed_ip) + db.fixed_ip_update(admin_ctxt, ip, {'allocated': True, + 'instance_id': instance_ref['id']}) + # FRAGILE: peeks at how the firewall names chains + chain_name = 'inst-%s' % instance_ref['id'] + + # create a firewall via setup_basic_filtering like libvirt_conn.spawn + # should have a chain with 0 rules + self.fw.setup_basic_filtering(instance_ref, network_info=nw_info) + self.assertTrue('provider' in self.fw.iptables.ipv4['filter'].chains) + rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules + if rule.chain == 'provider'] + self.assertEqual(0, len(rules)) + + # add a rule and send the update message, check for 1 rule + provider_fw0 = db.provider_fw_rule_create(admin_ctxt, + {'protocol': 'tcp', + 'cidr': '10.99.99.99/32', + 'from_port': 1, + 'to_port': 65535}) + self.fw.refresh_provider_fw_rules() + rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules + if rule.chain == 'provider'] + self.assertEqual(1, len(rules)) + + # Add another, refresh, and make sure number of rules goes to two + provider_fw1 = db.provider_fw_rule_create(admin_ctxt, + {'protocol': 'udp', + 'cidr': '10.99.99.99/32', + 'from_port': 1, + 'to_port': 65535}) + self.fw.refresh_provider_fw_rules() + rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules + if rule.chain == 'provider'] + self.assertEqual(2, len(rules)) + + # create the instance filter and make sure it has a jump rule + self.fw.prepare_instance_filter(instance_ref, network_info=nw_info) + self.fw.apply_instance_filter(instance_ref) + inst_rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules + if rule.chain == chain_name] + jump_rules = [rule for rule in inst_rules if '-j' in rule.rule] + provjump_rules = [] + # IptablesTable doesn't make rules unique internally + for rule in jump_rules: + if 'provider' in rule.rule and rule not in provjump_rules: + provjump_rules.append(rule) + self.assertEqual(1, len(provjump_rules)) + + # remove a rule from the db, cast to compute to refresh rule + db.provider_fw_rule_destroy(admin_ctxt, provider_fw1['id']) + self.fw.refresh_provider_fw_rules() + rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules + if rule.chain == 'provider'] + self.assertEqual(1, len(rules)) + class NWFilterTestCase(test.TestCase): def setUp(self): @@ -895,7 +1138,7 @@ class NWFilterTestCase(test.TestCase): self.fake_libvirt_connection = Mock() - self.fw = libvirt_conn.NWFilterFirewall( + self.fw = firewall.NWFilterFirewall( lambda: self.fake_libvirt_connection) def tearDown(self): @@ -1055,3 +1298,37 @@ class NWFilterTestCase(test.TestCase): network_info, "fake") self.assertEquals(len(result), 3) + + def test_unfilter_instance_undefines_nwfilters(self): + admin_ctxt = context.get_admin_context() + + fakefilter = NWFilterFakes() + self.fw._conn.nwfilterDefineXML = fakefilter.filterDefineXMLMock + self.fw._conn.nwfilterLookupByName = fakefilter.nwfilterLookupByName + + instance_ref = self._create_instance() + inst_id = instance_ref['id'] + + self.security_group = self.setup_and_return_security_group() + + db.instance_add_security_group(self.context, inst_id, + self.security_group.id) + + instance = db.instance_get(self.context, inst_id) + + ip = '10.11.12.13' + network_ref = db.project_get_network(self.context, 'fake') + fixed_ip = {'address': ip, 'network_id': network_ref['id']} + db.fixed_ip_create(admin_ctxt, fixed_ip) + db.fixed_ip_update(admin_ctxt, ip, {'allocated': True, + 'instance_id': inst_id}) + self.fw.setup_basic_filtering(instance) + self.fw.prepare_instance_filter(instance) + self.fw.apply_instance_filter(instance) + original_filter_count = len(fakefilter.filters) + self.fw.unfilter_instance(instance) + + # should undefine 2 filters: instance and instance-secgroup + self.assertEqual(original_filter_count - len(fakefilter.filters), 2) + + db.instance_destroy(admin_ctxt, instance_ref['id']) diff --git a/nova/tests/test_middleware.py b/nova/tests/test_middleware.py index 6564a6955..40d117c45 100644 --- a/nova/tests/test_middleware.py +++ b/nova/tests/test_middleware.py @@ -16,7 +16,6 @@ # License for the specific language governing permissions and limitations # under the License. -import datetime import webob import webob.dec import webob.exc diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py index cf8f4c05e..c5875a843 100644 --- a/nova/tests/test_misc.py +++ b/nova/tests/test_misc.py @@ -21,11 +21,24 @@ import select from eventlet import greenpool from eventlet import greenthread +from nova import exception from nova import test from nova import utils from nova.utils import parse_mailmap, str_dict_replace +class ExceptionTestCase(test.TestCase): + @staticmethod + def _raise_exc(exc): + raise exc() + + def test_exceptions_raise(self): + for name in dir(exception): + exc = getattr(exception, name) + if isinstance(exc, type): + self.assertRaises(exc, self._raise_exc, exc) + + class ProjectTestCase(test.TestCase): def test_authors_up_to_date(self): topdir = os.path.normpath(os.path.dirname(__file__) + '/../../') diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 77f6aaff3..9327c7129 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -18,7 +18,7 @@ """ Unit Tests for network code """ -import IPy +import netaddr import os from nova import test @@ -164,3 +164,33 @@ class IptablesManagerTestCase(test.TestCase): self.assertTrue('-A %s -j run_tests.py-%s' \ % (chain, chain) in new_lines, "Built-in chain %s not wrapped" % (chain,)) + + def test_will_empty_chain(self): + self.manager.ipv4['filter'].add_chain('test-chain') + self.manager.ipv4['filter'].add_rule('test-chain', '-j DROP') + old_count = len(self.manager.ipv4['filter'].rules) + self.manager.ipv4['filter'].empty_chain('test-chain') + self.assertEqual(old_count - 1, len(self.manager.ipv4['filter'].rules)) + + def test_will_empty_unwrapped_chain(self): + self.manager.ipv4['filter'].add_chain('test-chain', wrap=False) + self.manager.ipv4['filter'].add_rule('test-chain', '-j DROP', + wrap=False) + old_count = len(self.manager.ipv4['filter'].rules) + self.manager.ipv4['filter'].empty_chain('test-chain', wrap=False) + self.assertEqual(old_count - 1, len(self.manager.ipv4['filter'].rules)) + + def test_will_not_empty_wrapped_when_unwrapped(self): + self.manager.ipv4['filter'].add_chain('test-chain') + self.manager.ipv4['filter'].add_rule('test-chain', '-j DROP') + old_count = len(self.manager.ipv4['filter'].rules) + self.manager.ipv4['filter'].empty_chain('test-chain', wrap=False) + self.assertEqual(old_count, len(self.manager.ipv4['filter'].rules)) + + def test_will_not_empty_unwrapped_when_wrapped(self): + self.manager.ipv4['filter'].add_chain('test-chain', wrap=False) + self.manager.ipv4['filter'].add_rule('test-chain', '-j DROP', + wrap=False) + old_count = len(self.manager.ipv4['filter'].rules) + self.manager.ipv4['filter'].empty_chain('test-chain') + self.assertEqual(old_count, len(self.manager.ipv4['filter'].rules)) diff --git a/nova/tests/test_notifier.py b/nova/tests/test_notifier.py index b6b0fcc68..64b799a2c 100644 --- a/nova/tests/test_notifier.py +++ b/nova/tests/test_notifier.py @@ -13,10 +13,12 @@ # License for the specific language governing permissions and limitations # under the License. -import nova +import stubout +import nova from nova import context from nova import flags +from nova import log from nova import rpc import nova.notifier.api from nova.notifier.api import notify @@ -24,8 +26,6 @@ from nova.notifier import no_op_notifier from nova.notifier import rabbit_notifier from nova import test -import stubout - class NotifierTestCase(test.TestCase): """Test case for notifications""" @@ -115,3 +115,22 @@ class NotifierTestCase(test.TestCase): notify('publisher_id', 'event_type', 'DEBUG', dict(a=3)) self.assertEqual(self.test_topic, 'testnotify.debug') + + def test_error_notification(self): + self.stubs.Set(nova.flags.FLAGS, 'notification_driver', + 'nova.notifier.rabbit_notifier') + self.stubs.Set(nova.flags.FLAGS, 'publish_errors', True) + LOG = log.getLogger('nova') + LOG.setup_from_flags() + msgs = [] + + def mock_cast(context, topic, data): + msgs.append(data) + + self.stubs.Set(nova.rpc, 'cast', mock_cast) + LOG.error('foo') + self.assertEqual(1, len(msgs)) + msg = msgs[0] + self.assertEqual(msg['event_type'], 'error_notification') + self.assertEqual(msg['priority'], 'ERROR') + self.assertEqual(msg['payload']['error'], 'foo') diff --git a/nova/tests/test_quota.py b/nova/tests/test_quota.py index 7ace2ad7d..0691231e4 100644 --- a/nova/tests/test_quota.py +++ b/nova/tests/test_quota.py @@ -104,6 +104,10 @@ class QuotaTestCase(test.TestCase): num_instances = quota.allowed_instances(self.context, 100, self._get_instance_type('m1.small')) self.assertEqual(num_instances, 10) + db.quota_create(self.context, self.project.id, 'ram', 3 * 2048) + num_instances = quota.allowed_instances(self.context, 100, + self._get_instance_type('m1.small')) + self.assertEqual(num_instances, 3) # metadata_items too_many_items = FLAGS.quota_metadata_items + 1000 @@ -120,7 +124,8 @@ class QuotaTestCase(test.TestCase): def test_unlimited_instances(self): FLAGS.quota_instances = 2 - FLAGS.quota_cores = 1000 + FLAGS.quota_ram = -1 + FLAGS.quota_cores = -1 instance_type = self._get_instance_type('m1.small') num_instances = quota.allowed_instances(self.context, 100, instance_type) @@ -133,8 +138,25 @@ class QuotaTestCase(test.TestCase): instance_type) self.assertEqual(num_instances, 101) + def test_unlimited_ram(self): + FLAGS.quota_instances = -1 + FLAGS.quota_ram = 2 * 2048 + FLAGS.quota_cores = -1 + instance_type = self._get_instance_type('m1.small') + num_instances = quota.allowed_instances(self.context, 100, + instance_type) + self.assertEqual(num_instances, 2) + db.quota_create(self.context, self.project.id, 'ram', None) + num_instances = quota.allowed_instances(self.context, 100, + instance_type) + self.assertEqual(num_instances, 100) + num_instances = quota.allowed_instances(self.context, 101, + instance_type) + self.assertEqual(num_instances, 101) + def test_unlimited_cores(self): - FLAGS.quota_instances = 1000 + FLAGS.quota_instances = -1 + FLAGS.quota_ram = -1 FLAGS.quota_cores = 2 instance_type = self._get_instance_type('m1.small') num_instances = quota.allowed_instances(self.context, 100, @@ -150,7 +172,7 @@ class QuotaTestCase(test.TestCase): def test_unlimited_volumes(self): FLAGS.quota_volumes = 10 - FLAGS.quota_gigabytes = 1000 + FLAGS.quota_gigabytes = -1 volumes = quota.allowed_volumes(self.context, 100, 1) self.assertEqual(volumes, 10) db.quota_create(self.context, self.project.id, 'volumes', None) @@ -160,7 +182,7 @@ class QuotaTestCase(test.TestCase): self.assertEqual(volumes, 101) def test_unlimited_gigabytes(self): - FLAGS.quota_volumes = 1000 + FLAGS.quota_volumes = -1 FLAGS.quota_gigabytes = 10 volumes = quota.allowed_volumes(self.context, 100, 1) self.assertEqual(volumes, 10) @@ -201,7 +223,7 @@ class QuotaTestCase(test.TestCase): min_count=1, max_count=1, instance_type=inst_type, - image_id=1) + image_href=1) for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -215,7 +237,7 @@ class QuotaTestCase(test.TestCase): min_count=1, max_count=1, instance_type=inst_type, - image_id=1) + image_href=1) for instance_id in instance_ids: db.instance_destroy(self.context, instance_id) @@ -228,6 +250,7 @@ class QuotaTestCase(test.TestCase): volume.API().create, self.context, size=10, + snapshot_id=None, name='', description='') for volume_id in volume_ids: @@ -241,6 +264,7 @@ class QuotaTestCase(test.TestCase): volume.API().create, self.context, size=10, + snapshot_id=None, name='', description='') for volume_id in volume_ids: @@ -271,25 +295,64 @@ class QuotaTestCase(test.TestCase): min_count=1, max_count=1, instance_type=inst_type, - image_id='fake', + image_href='fake', metadata=metadata) - def test_allowed_injected_files(self): - self.assertEqual( - quota.allowed_injected_files(self.context), - FLAGS.quota_max_injected_files) + def test_default_allowed_injected_files(self): + FLAGS.quota_max_injected_files = 55 + self.assertEqual(quota.allowed_injected_files(self.context, 100), 55) + + def test_overridden_allowed_injected_files(self): + FLAGS.quota_max_injected_files = 5 + db.quota_create(self.context, self.project.id, 'injected_files', 77) + self.assertEqual(quota.allowed_injected_files(self.context, 100), 77) + + def test_unlimited_default_allowed_injected_files(self): + FLAGS.quota_max_injected_files = -1 + self.assertEqual(quota.allowed_injected_files(self.context, 100), 100) + + def test_unlimited_db_allowed_injected_files(self): + FLAGS.quota_max_injected_files = 5 + db.quota_create(self.context, self.project.id, 'injected_files', None) + self.assertEqual(quota.allowed_injected_files(self.context, 100), 100) + + def test_default_allowed_injected_file_content_bytes(self): + FLAGS.quota_max_injected_file_content_bytes = 12345 + limit = quota.allowed_injected_file_content_bytes(self.context, 23456) + self.assertEqual(limit, 12345) + + def test_overridden_allowed_injected_file_content_bytes(self): + FLAGS.quota_max_injected_file_content_bytes = 12345 + db.quota_create(self.context, self.project.id, + 'injected_file_content_bytes', 5678) + limit = quota.allowed_injected_file_content_bytes(self.context, 23456) + self.assertEqual(limit, 5678) + + def test_unlimited_default_allowed_injected_file_content_bytes(self): + FLAGS.quota_max_injected_file_content_bytes = -1 + limit = quota.allowed_injected_file_content_bytes(self.context, 23456) + self.assertEqual(limit, 23456) + + def test_unlimited_db_allowed_injected_file_content_bytes(self): + FLAGS.quota_max_injected_file_content_bytes = 12345 + db.quota_create(self.context, self.project.id, + 'injected_file_content_bytes', None) + limit = quota.allowed_injected_file_content_bytes(self.context, 23456) + self.assertEqual(limit, 23456) def _create_with_injected_files(self, files): + FLAGS.image_service = 'nova.image.fake.FakeImageService' api = compute.API(image_service=self.StubImageService()) inst_type = instance_types.get_instance_type_by_name('m1.small') api.create(self.context, min_count=1, max_count=1, - instance_type=inst_type, image_id='fake', + instance_type=inst_type, image_href='3', injected_files=files) def test_no_injected_files(self): + FLAGS.image_service = 'nova.image.fake.FakeImageService' api = compute.API(image_service=self.StubImageService()) inst_type = instance_types.get_instance_type_by_name('m1.small') - api.create(self.context, instance_type=inst_type, image_id='fake') + api.create(self.context, instance_type=inst_type, image_href='3') def test_max_injected_files(self): files = [] @@ -304,11 +367,6 @@ class QuotaTestCase(test.TestCase): self.assertRaises(quota.QuotaError, self._create_with_injected_files, files) - def test_allowed_injected_file_content_bytes(self): - self.assertEqual( - quota.allowed_injected_file_content_bytes(self.context), - FLAGS.quota_max_injected_file_content_bytes) - def test_max_injected_file_content_bytes(self): max = FLAGS.quota_max_injected_file_content_bytes content = ''.join(['a' for i in xrange(max)]) diff --git a/nova/tests/test_rpc.py b/nova/tests/test_rpc.py index 44d7c91eb..ffd748efe 100644 --- a/nova/tests/test_rpc.py +++ b/nova/tests/test_rpc.py @@ -31,7 +31,6 @@ LOG = logging.getLogger('nova.tests.rpc') class RpcTestCase(test.TestCase): - """Test cases for rpc""" def setUp(self): super(RpcTestCase, self).setUp() self.conn = rpc.Connection.instance(True) @@ -43,14 +42,55 @@ class RpcTestCase(test.TestCase): self.context = context.get_admin_context() def test_call_succeed(self): - """Get a value through rpc call""" value = 42 result = rpc.call(self.context, 'test', {"method": "echo", "args": {"value": value}}) self.assertEqual(value, result) + def test_call_succeed_despite_multiple_returns(self): + value = 42 + result = rpc.call(self.context, 'test', {"method": "echo_three_times", + "args": {"value": value}}) + self.assertEqual(value + 2, result) + + def test_call_succeed_despite_multiple_returns_yield(self): + value = 42 + result = rpc.call(self.context, 'test', + {"method": "echo_three_times_yield", + "args": {"value": value}}) + self.assertEqual(value + 2, result) + + def test_multicall_succeed_once(self): + value = 42 + result = rpc.multicall(self.context, + 'test', + {"method": "echo", + "args": {"value": value}}) + for i, x in enumerate(result): + if i > 0: + self.fail('should only receive one response') + self.assertEqual(value + i, x) + + def test_multicall_succeed_three_times(self): + value = 42 + result = rpc.multicall(self.context, + 'test', + {"method": "echo_three_times", + "args": {"value": value}}) + for i, x in enumerate(result): + self.assertEqual(value + i, x) + + def test_multicall_succeed_three_times_yield(self): + value = 42 + result = rpc.multicall(self.context, + 'test', + {"method": "echo_three_times_yield", + "args": {"value": value}}) + for i, x in enumerate(result): + self.assertEqual(value + i, x) + def test_context_passed(self): - """Makes sure a context is passed through rpc call""" + """Makes sure a context is passed through rpc call.""" value = 42 result = rpc.call(self.context, 'test', {"method": "context", @@ -58,11 +98,12 @@ class RpcTestCase(test.TestCase): self.assertEqual(self.context.to_dict(), result) def test_call_exception(self): - """Test that exception gets passed back properly + """Test that exception gets passed back properly. rpc.call returns a RemoteError object. The value of the exception is converted to a string, so we convert it back to an int in the test. + """ value = 42 self.assertRaises(rpc.RemoteError, @@ -81,7 +122,7 @@ class RpcTestCase(test.TestCase): self.assertEqual(int(exc.value), value) def test_nested_calls(self): - """Test that we can do an rpc.call inside another call""" + """Test that we can do an rpc.call inside another call.""" class Nested(object): @staticmethod def echo(context, queue, value): @@ -108,25 +149,80 @@ class RpcTestCase(test.TestCase): "value": value}}) self.assertEqual(value, result) + def test_connectionpool_single(self): + """Test that ConnectionPool recycles a single connection.""" + conn1 = rpc.ConnectionPool.get() + rpc.ConnectionPool.put(conn1) + conn2 = rpc.ConnectionPool.get() + rpc.ConnectionPool.put(conn2) + self.assertEqual(conn1, conn2) + + def test_connectionpool_double(self): + """Test that ConnectionPool returns and reuses separate connections. + + When called consecutively we should get separate connections and upon + returning them those connections should be reused for future calls + before generating a new connection. + + """ + conn1 = rpc.ConnectionPool.get() + conn2 = rpc.ConnectionPool.get() + + self.assertNotEqual(conn1, conn2) + rpc.ConnectionPool.put(conn1) + rpc.ConnectionPool.put(conn2) + + conn3 = rpc.ConnectionPool.get() + conn4 = rpc.ConnectionPool.get() + self.assertEqual(conn1, conn3) + self.assertEqual(conn2, conn4) + + def test_connectionpool_limit(self): + """Test connection pool limit and connection uniqueness.""" + max_size = FLAGS.rpc_conn_pool_size + conns = [] + + for i in xrange(max_size): + conns.append(rpc.ConnectionPool.get()) + + self.assertFalse(rpc.ConnectionPool.free_items) + self.assertEqual(rpc.ConnectionPool.current_size, + rpc.ConnectionPool.max_size) + self.assertEqual(len(set(conns)), max_size) + class TestReceiver(object): - """Simple Proxy class so the consumer has methods to call + """Simple Proxy class so the consumer has methods to call. + + Uses static methods because we aren't actually storing any state. - Uses static methods because we aren't actually storing any state""" + """ @staticmethod def echo(context, value): - """Simply returns whatever value is sent in""" + """Simply returns whatever value is sent in.""" LOG.debug(_("Received %s"), value) return value @staticmethod def context(context, value): - """Returns dictionary version of context""" + """Returns dictionary version of context.""" LOG.debug(_("Received %s"), context) return context.to_dict() @staticmethod + def echo_three_times(context, value): + context.reply(value) + context.reply(value + 1) + context.reply(value + 2) + + @staticmethod + def echo_three_times_yield(context, value): + yield value + yield value + 1 + yield value + 2 + + @staticmethod def fail(context, value): - """Raises an exception with the value sent in""" + """Raises an exception with the value sent in.""" raise Exception(value) diff --git a/nova/tests/test_service.py b/nova/tests/test_service.py index d48de2057..d1cc8bd61 100644 --- a/nova/tests/test_service.py +++ b/nova/tests/test_service.py @@ -106,7 +106,10 @@ class ServiceTestCase(test.TestCase): # NOTE(vish): Create was moved out of mox replay to make sure that # the looping calls are created in StartService. - app = service.Service.create(host=host, binary=binary) + app = service.Service.create(host=host, binary=binary, topic=topic) + + self.mox.StubOutWithMock(service.rpc.Connection, 'instance') + service.rpc.Connection.instance(new=mox.IgnoreArg()) self.mox.StubOutWithMock(rpc, 'TopicAdapterConsumer', @@ -114,6 +117,11 @@ class ServiceTestCase(test.TestCase): self.mox.StubOutWithMock(rpc, 'FanoutAdapterConsumer', use_mock_anything=True) + + self.mox.StubOutWithMock(rpc, + 'ConsumerSet', + use_mock_anything=True) + rpc.TopicAdapterConsumer(connection=mox.IgnoreArg(), topic=topic, proxy=mox.IsA(service.Service)).AndReturn( @@ -129,9 +137,14 @@ class ServiceTestCase(test.TestCase): proxy=mox.IsA(service.Service)).AndReturn( rpc.FanoutAdapterConsumer) - rpc.TopicAdapterConsumer.attach_to_eventlet() - rpc.TopicAdapterConsumer.attach_to_eventlet() - rpc.FanoutAdapterConsumer.attach_to_eventlet() + def wait_func(self, limit=None): + return None + + mock_cset = self.mox.CreateMock(rpc.ConsumerSet, + {'wait': wait_func}) + rpc.ConsumerSet(connection=mox.IgnoreArg(), + consumer_list=mox.IsA(list)).AndReturn(mock_cset) + wait_func(mox.IgnoreArg()) service_create = {'host': host, 'binary': binary, @@ -287,8 +300,42 @@ class ServiceTestCase(test.TestCase): # Creating mocks self.mox.StubOutWithMock(service.rpc.Connection, 'instance') service.rpc.Connection.instance(new=mox.IgnoreArg()) - service.rpc.Connection.instance(new=mox.IgnoreArg()) - service.rpc.Connection.instance(new=mox.IgnoreArg()) + + self.mox.StubOutWithMock(rpc, + 'TopicAdapterConsumer', + use_mock_anything=True) + self.mox.StubOutWithMock(rpc, + 'FanoutAdapterConsumer', + use_mock_anything=True) + + self.mox.StubOutWithMock(rpc, + 'ConsumerSet', + use_mock_anything=True) + + rpc.TopicAdapterConsumer(connection=mox.IgnoreArg(), + topic=topic, + proxy=mox.IsA(service.Service)).AndReturn( + rpc.TopicAdapterConsumer) + + rpc.TopicAdapterConsumer(connection=mox.IgnoreArg(), + topic='%s.%s' % (topic, host), + proxy=mox.IsA(service.Service)).AndReturn( + rpc.TopicAdapterConsumer) + + rpc.FanoutAdapterConsumer(connection=mox.IgnoreArg(), + topic=topic, + proxy=mox.IsA(service.Service)).AndReturn( + rpc.FanoutAdapterConsumer) + + def wait_func(self, limit=None): + return None + + mock_cset = self.mox.CreateMock(rpc.ConsumerSet, + {'wait': wait_func}) + rpc.ConsumerSet(connection=mox.IgnoreArg(), + consumer_list=mox.IsA(list)).AndReturn(mock_cset) + wait_func(mox.IgnoreArg()) + self.mox.StubOutWithMock(serv.manager.driver, 'update_available_resource') serv.manager.driver.update_available_resource(mox.IgnoreArg(), host) diff --git a/nova/tests/test_utils.py b/nova/tests/test_utils.py index 8f7e83c3e..3a3f914e4 100644 --- a/nova/tests/test_utils.py +++ b/nova/tests/test_utils.py @@ -275,3 +275,21 @@ class GenericUtilsTestCase(test.TestCase): # error case result = utils.parse_server_string('www.exa:mple.com:8443') self.assertEqual(('', ''), result) + + +class IsUUIDLikeTestCase(test.TestCase): + def assertUUIDLike(self, val, expected): + result = utils.is_uuid_like(val) + self.assertEqual(result, expected) + + def test_good_uuid(self): + val = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' + self.assertUUIDLike(val, True) + + def test_integer_passed(self): + val = 1 + self.assertUUIDLike(val, False) + + def test_non_uuid_string_passed(self): + val = 'foo-fooo' + self.assertUUIDLike(val, False) diff --git a/nova/tests/test_vlan_network.py b/nova/tests/test_vlan_network.py index 063b81832..a1c8ab11c 100644 --- a/nova/tests/test_vlan_network.py +++ b/nova/tests/test_vlan_network.py @@ -18,7 +18,7 @@ """ Unit Tests for vlan network code """ -import IPy +import netaddr import os from nova import context @@ -44,8 +44,8 @@ class VlanNetworkTestCase(base.NetworkTestCase): # TODO(vish): better way of adding floating ips self.context._project = self.projects[0] self.context.project_id = self.projects[0].id - pubnet = IPy.IP(flags.FLAGS.floating_range) - address = str(pubnet[0]) + pubnet = netaddr.IPNetwork(flags.FLAGS.floating_range) + address = str(list(pubnet)[0]) try: db.floating_ip_get_by_address(context.get_admin_context(), address) except exception.NotFound: diff --git a/nova/tests/test_vmwareapi.py b/nova/tests/test_vmwareapi.py index 22b66010a..eddf01e9f 100644 --- a/nova/tests/test_vmwareapi.py +++ b/nova/tests/test_vmwareapi.py @@ -55,8 +55,7 @@ class VMWareAPIVMTestCase(test.TestCase): vmwareapi_fake.reset()
db_fakes.stub_out_db_instance_api(self.stubs)
stubs.set_stubs(self.stubs)
- glance_stubs.stubout_glance_client(self.stubs,
- glance_stubs.FakeGlance)
+ glance_stubs.stubout_glance_client(self.stubs)
self.conn = vmwareapi_conn.get_connection(False)
def _create_instance_in_the_db(self):
@@ -64,13 +63,13 @@ class VMWareAPIVMTestCase(test.TestCase): 'id': 1,
'project_id': self.project.id,
'user_id': self.user.id,
- 'image_id': "1",
+ 'image_ref': "1",
'kernel_id': "1",
'ramdisk_id': "1",
'instance_type': 'm1.large',
'mac_address': 'aa:bb:cc:dd:ee:ff',
}
- self.instance = db.instance_create(values)
+ self.instance = db.instance_create(None, values)
def _create_vm(self):
"""Create and spawn the VM."""
diff --git a/nova/tests/test_volume.py b/nova/tests/test_volume.py index 236d12434..4f10ee6af 100644 --- a/nova/tests/test_volume.py +++ b/nova/tests/test_volume.py @@ -45,10 +45,11 @@ class VolumeTestCase(test.TestCase): self.context = context.get_admin_context() @staticmethod - def _create_volume(size='0'): + def _create_volume(size='0', snapshot_id=None): """Create a volume object.""" vol = {} vol['size'] = size + vol['snapshot_id'] = snapshot_id vol['user_id'] = 'fake' vol['project_id'] = 'fake' vol['availability_zone'] = FLAGS.storage_availability_zone @@ -69,6 +70,25 @@ class VolumeTestCase(test.TestCase): self.context, volume_id) + def test_create_volume_from_snapshot(self): + """Test volume can be created from a snapshot.""" + volume_src_id = self._create_volume() + self.volume.create_volume(self.context, volume_src_id) + snapshot_id = self._create_snapshot(volume_src_id) + self.volume.create_snapshot(self.context, volume_src_id, snapshot_id) + volume_dst_id = self._create_volume(0, snapshot_id) + self.volume.create_volume(self.context, volume_dst_id, snapshot_id) + self.assertEqual(volume_dst_id, db.volume_get( + context.get_admin_context(), + volume_dst_id).id) + self.assertEqual(snapshot_id, db.volume_get( + context.get_admin_context(), + volume_dst_id).snapshot_id) + + self.volume.delete_volume(self.context, volume_dst_id) + self.volume.delete_snapshot(self.context, snapshot_id) + self.volume.delete_volume(self.context, volume_src_id) + def test_too_big_volume(self): """Ensure failure if a too large of a volume is requested.""" # FIXME(vish): validation needs to move into the data layer in @@ -176,6 +196,34 @@ class VolumeTestCase(test.TestCase): # This will allow us to test cross-node interactions pass + @staticmethod + def _create_snapshot(volume_id, size='0'): + """Create a snapshot object.""" + snap = {} + snap['volume_size'] = size + snap['user_id'] = 'fake' + snap['project_id'] = 'fake' + snap['volume_id'] = volume_id + snap['status'] = "creating" + return db.snapshot_create(context.get_admin_context(), snap)['id'] + + def test_create_delete_snapshot(self): + """Test snapshot can be created and deleted.""" + volume_id = self._create_volume() + self.volume.create_volume(self.context, volume_id) + snapshot_id = self._create_snapshot(volume_id) + self.volume.create_snapshot(self.context, volume_id, snapshot_id) + self.assertEqual(snapshot_id, + db.snapshot_get(context.get_admin_context(), + snapshot_id).id) + + self.volume.delete_snapshot(self.context, snapshot_id) + self.assertRaises(exception.NotFound, + db.snapshot_get, + self.context, + snapshot_id) + self.volume.delete_volume(self.context, volume_id) + class DriverTestCase(test.TestCase): """Base Test class for Drivers.""" diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index be1e35697..d9a514745 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -33,12 +33,12 @@ from nova import utils from nova.auth import manager from nova.compute import instance_types from nova.compute import power_state +from nova import exception from nova.virt import xenapi_conn from nova.virt.xenapi import fake as xenapi_fake from nova.virt.xenapi import volume_utils +from nova.virt.xenapi import vmops from nova.virt.xenapi import vm_utils -from nova.virt.xenapi.vmops import SimpleDH -from nova.virt.xenapi.vmops import VMOps from nova.tests.db import fakes as db_fakes from nova.tests.xenapi import stubs from nova.tests.glance import stubs as glance_stubs @@ -79,12 +79,13 @@ class XenAPIVolumeTestCase(test.TestCase): self.values = {'id': 1, 'project_id': 'fake', 'user_id': 'fake', - 'image_id': 1, + 'image_ref': 1, 'kernel_id': 2, 'ramdisk_id': 3, 'instance_type_id': '3', # m1.large 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': 'linux'} + 'os_type': 'linux', + 'architecture': 'x86-64'} def _create_volume(self, size='0'): """Create a volume object.""" @@ -191,10 +192,9 @@ class XenAPIVMTestCase(test.TestCase): stubs.stubout_get_this_vm_uuid(self.stubs) stubs.stubout_stream_disk(self.stubs) stubs.stubout_is_vdi_pv(self.stubs) - self.stubs.Set(VMOps, 'reset_network', reset_network) + self.stubs.Set(vmops.VMOps, 'reset_network', reset_network) stubs.stub_out_vm_methods(self.stubs) - glance_stubs.stubout_glance_client(self.stubs, - glance_stubs.FakeGlance) + glance_stubs.stubout_glance_client(self.stubs) fake_utils.stub_out_utils_execute(self.stubs) self.context = context.RequestContext('fake', 'fake', False) self.conn = xenapi_conn.get_connection(False) @@ -207,12 +207,13 @@ class XenAPIVMTestCase(test.TestCase): 'id': id, 'project_id': proj, 'user_id': user, - 'image_id': 1, + 'image_ref': 1, 'kernel_id': 2, 'ramdisk_id': 3, 'instance_type_id': '3', # m1.large 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': 'linux'} + 'os_type': 'linux', + 'architecture': 'x86-64'} instance = db.instance_create(self.context, values) self.conn.spawn(instance) @@ -229,6 +230,23 @@ class XenAPIVMTestCase(test.TestCase): instance = self._create_instance() self.conn.get_diagnostics(instance) + def test_instance_snapshot_fails_with_no_primary_vdi(self): + def create_bad_vbd(vm_ref, vdi_ref): + vbd_rec = {'VM': vm_ref, + 'VDI': vdi_ref, + 'userdevice': 'fake', + 'currently_attached': False} + vbd_ref = xenapi_fake._create_object('VBD', vbd_rec) + xenapi_fake.after_VBD_create(vbd_ref, vbd_rec) + return vbd_ref + + self.stubs.Set(xenapi_fake, 'create_vbd', create_bad_vbd) + stubs.stubout_instance_snapshot(self.stubs) + instance = self._create_instance() + + name = "MySnapshot" + self.assertRaises(exception.Error, self.conn.snapshot, instance, name) + def test_instance_snapshot(self): stubs.stubout_instance_snapshot(self.stubs) instance = self._create_instance() @@ -332,7 +350,7 @@ class XenAPIVMTestCase(test.TestCase): def check_vm_params_for_linux(self): self.assertEquals(self.vm['platform']['nx'], 'false') - self.assertEquals(self.vm['PV_args'], 'clocksource=jiffies') + self.assertEquals(self.vm['PV_args'], '') self.assertEquals(self.vm['PV_bootloader'], 'pygrub') # check that these are not set @@ -351,23 +369,27 @@ class XenAPIVMTestCase(test.TestCase): self.assertEquals(self.vm['HVM_boot_params'], {}) self.assertEquals(self.vm['HVM_boot_policy'], '') - def _test_spawn(self, image_id, kernel_id, ramdisk_id, + def _test_spawn(self, image_ref, kernel_id, ramdisk_id, instance_type_id="3", os_type="linux", - instance_id=1, check_injection=False): + architecture="x86-64", instance_id=1, + check_injection=False): stubs.stubout_loopingcall_start(self.stubs) values = {'id': instance_id, 'project_id': self.project.id, 'user_id': self.user.id, - 'image_id': image_id, + 'image_ref': image_ref, 'kernel_id': kernel_id, 'ramdisk_id': ramdisk_id, 'instance_type_id': instance_type_id, 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': os_type} + 'os_type': os_type, + 'architecture': architecture} instance = db.instance_create(self.context, values) self.conn.spawn(instance) self.create_vm_record(self.conn, os_type, instance_id) self.check_vm_record(self.conn, check_injection) + self.assertTrue(instance.os_type) + self.assertTrue(instance.architecture) def test_spawn_not_enough_memory(self): FLAGS.xenapi_image_service = 'glance' @@ -392,13 +414,36 @@ class XenAPIVMTestCase(test.TestCase): def test_spawn_vhd_glance_linux(self): FLAGS.xenapi_image_service = 'glance' self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, - os_type="linux") + os_type="linux", architecture="x86-64") self.check_vm_params_for_linux() + def test_spawn_vhd_glance_swapdisk(self): + # Change the default host_call_plugin to one that'll return + # a swap disk + orig_func = stubs.FakeSessionForVMTests.host_call_plugin + + stubs.FakeSessionForVMTests.host_call_plugin = \ + stubs.FakeSessionForVMTests.host_call_plugin_swap + + try: + # We'll steal the above glance linux test + self.test_spawn_vhd_glance_linux() + finally: + # Make sure to put this back + stubs.FakeSessionForVMTests.host_call_plugin = orig_func + + # We should have 2 VBDs. + self.assertEqual(len(self.vm['VBDs']), 2) + # Now test that we have 1. + self.tearDown() + self.setUp() + self.test_spawn_vhd_glance_linux() + self.assertEqual(len(self.vm['VBDs']), 1) + def test_spawn_vhd_glance_windows(self): FLAGS.xenapi_image_service = 'glance' self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, - os_type="windows") + os_type="windows", architecture="i386") self.check_vm_params_for_windows() def test_spawn_glance(self): @@ -544,12 +589,13 @@ class XenAPIVMTestCase(test.TestCase): 'id': 1, 'project_id': self.project.id, 'user_id': self.user.id, - 'image_id': 1, + 'image_ref': 1, 'kernel_id': 2, 'ramdisk_id': 3, 'instance_type_id': '3', # m1.large 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': 'linux'} + 'os_type': 'linux', + 'architecture': 'x86-64'} instance = db.instance_create(self.context, values) self.conn.spawn(instance) return instance @@ -559,8 +605,8 @@ class XenAPIDiffieHellmanTestCase(test.TestCase): """Unit tests for Diffie-Hellman code.""" def setUp(self): super(XenAPIDiffieHellmanTestCase, self).setUp() - self.alice = SimpleDH() - self.bob = SimpleDH() + self.alice = vmops.SimpleDH() + self.bob = vmops.SimpleDH() def test_shared(self): alice_pub = self.alice.get_public() @@ -569,11 +615,29 @@ class XenAPIDiffieHellmanTestCase(test.TestCase): bob_shared = self.bob.compute_shared(alice_pub) self.assertEquals(alice_shared, bob_shared) - def test_encryption(self): - msg = "This is a top-secret message" - enc = self.alice.encrypt(msg) + def _test_encryption(self, message): + enc = self.alice.encrypt(message) + self.assertFalse(enc.endswith('\n')) dec = self.bob.decrypt(enc) - self.assertEquals(dec, msg) + self.assertEquals(dec, message) + + def test_encrypt_simple_message(self): + self._test_encryption('This is a simple message.') + + def test_encrypt_message_with_newlines_at_end(self): + self._test_encryption('This message has a newline at the end.\n') + + def test_encrypt_many_newlines_at_end(self): + self._test_encryption('Message with lotsa newlines.\n\n\n') + + def test_encrypt_newlines_inside_message(self): + self._test_encryption('Message\nwith\ninterior\nnewlines.') + + def test_encrypt_with_leading_newlines(self): + self._test_encryption('\n\nMessage with leading newlines.') + + def test_encrypt_really_long_message(self): + self._test_encryption(''.join(['abcd' for i in xrange(1024)])) def tearDown(self): super(XenAPIDiffieHellmanTestCase, self).tearDown() @@ -600,19 +664,19 @@ class XenAPIMigrateInstance(test.TestCase): self.values = {'id': 1, 'project_id': self.project.id, 'user_id': self.user.id, - 'image_id': 1, + 'image_ref': 1, 'kernel_id': None, 'ramdisk_id': None, 'local_gb': 5, 'instance_type_id': '3', # m1.large 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': 'linux'} + 'os_type': 'linux', + 'architecture': 'x86-64'} fake_utils.stub_out_utils_execute(self.stubs) stubs.stub_out_migration_methods(self.stubs) stubs.stubout_get_this_vm_uuid(self.stubs) - glance_stubs.stubout_glance_client(self.stubs, - glance_stubs.FakeGlance) + glance_stubs.stubout_glance_client(self.stubs) def tearDown(self): super(XenAPIMigrateInstance, self).tearDown() @@ -638,8 +702,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): """Unit tests for code that detects the ImageType.""" def setUp(self): super(XenAPIDetermineDiskImageTestCase, self).setUp() - glance_stubs.stubout_glance_client(self.stubs, - glance_stubs.FakeGlance) + glance_stubs.stubout_glance_client(self.stubs) class FakeInstance(object): pass @@ -647,6 +710,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): self.fake_instance = FakeInstance() self.fake_instance.id = 42 self.fake_instance.os_type = 'linux' + self.fake_instance.architecture = 'x86-64' def assert_disk_type(self, disk_type): dt = vm_utils.VMHelper.determine_disk_image_type( @@ -656,7 +720,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): def test_instance_disk(self): """If a kernel is specified, the image type is DISK (aka machine).""" FLAGS.xenapi_image_service = 'objectstore' - self.fake_instance.image_id = glance_stubs.FakeGlance.IMAGE_MACHINE + self.fake_instance.image_ref = glance_stubs.FakeGlance.IMAGE_MACHINE self.fake_instance.kernel_id = glance_stubs.FakeGlance.IMAGE_KERNEL self.assert_disk_type(vm_utils.ImageType.DISK) @@ -666,7 +730,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): DISK_RAW is assumed. """ FLAGS.xenapi_image_service = 'objectstore' - self.fake_instance.image_id = glance_stubs.FakeGlance.IMAGE_RAW + self.fake_instance.image_ref = glance_stubs.FakeGlance.IMAGE_RAW self.fake_instance.kernel_id = None self.assert_disk_type(vm_utils.ImageType.DISK_RAW) @@ -676,7 +740,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): this case will be 'raw'. """ FLAGS.xenapi_image_service = 'glance' - self.fake_instance.image_id = glance_stubs.FakeGlance.IMAGE_RAW + self.fake_instance.image_ref = glance_stubs.FakeGlance.IMAGE_RAW self.fake_instance.kernel_id = None self.assert_disk_type(vm_utils.ImageType.DISK_RAW) @@ -686,11 +750,33 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): this case will be 'vhd'. """ FLAGS.xenapi_image_service = 'glance' - self.fake_instance.image_id = glance_stubs.FakeGlance.IMAGE_VHD + self.fake_instance.image_ref = glance_stubs.FakeGlance.IMAGE_VHD self.fake_instance.kernel_id = None self.assert_disk_type(vm_utils.ImageType.DISK_VHD) +class CompareVersionTestCase(test.TestCase): + def test_less_than(self): + """Test that cmp_version compares a as less than b""" + self.assertTrue(vmops.cmp_version('1.2.3.4', '1.2.3.5') < 0) + + def test_greater_than(self): + """Test that cmp_version compares a as greater than b""" + self.assertTrue(vmops.cmp_version('1.2.3.5', '1.2.3.4') > 0) + + def test_equal(self): + """Test that cmp_version compares a as equal to b""" + self.assertTrue(vmops.cmp_version('1.2.3.4', '1.2.3.4') == 0) + + def test_non_lexical(self): + """Test that cmp_version compares non-lexically""" + self.assertTrue(vmops.cmp_version('1.2.3.10', '1.2.3.4') > 0) + + def test_length(self): + """Test that cmp_version compares by length as last resort""" + self.assertTrue(vmops.cmp_version('1.2.3', '1.2.3.4') < 0) + + class FakeXenApi(object): """Fake XenApi for testing HostState.""" diff --git a/nova/tests/test_zone_aware_scheduler.py b/nova/tests/test_zone_aware_scheduler.py deleted file mode 100644 index fdcde34c9..000000000 --- a/nova/tests/test_zone_aware_scheduler.py +++ /dev/null @@ -1,119 +0,0 @@ -# Copyright 2011 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -""" -Tests For Zone Aware Scheduler. -""" - -from nova import test -from nova.scheduler import driver -from nova.scheduler import zone_aware_scheduler -from nova.scheduler import zone_manager - - -class FakeZoneAwareScheduler(zone_aware_scheduler.ZoneAwareScheduler): - def filter_hosts(self, num, specs): - # NOTE(sirp): this is returning [(hostname, services)] - return self.zone_manager.service_states.items() - - def weigh_hosts(self, num, specs, hosts): - fake_weight = 99 - weighted = [] - for hostname, caps in hosts: - weighted.append(dict(weight=fake_weight, name=hostname)) - return weighted - - -class FakeZoneManager(zone_manager.ZoneManager): - def __init__(self): - self.service_states = { - 'host1': { - 'compute': {'ram': 1000} - }, - 'host2': { - 'compute': {'ram': 2000} - }, - 'host3': { - 'compute': {'ram': 3000} - } - } - - -class FakeEmptyZoneManager(zone_manager.ZoneManager): - def __init__(self): - self.service_states = {} - - -def fake_empty_call_zone_method(context, method, specs): - return [] - - -def fake_call_zone_method(context, method, specs): - return [ - ('zone1', [ - dict(weight=1, blob='AAAAAAA'), - dict(weight=111, blob='BBBBBBB'), - dict(weight=112, blob='CCCCCCC'), - dict(weight=113, blob='DDDDDDD'), - ]), - ('zone2', [ - dict(weight=120, blob='EEEEEEE'), - dict(weight=2, blob='FFFFFFF'), - dict(weight=122, blob='GGGGGGG'), - dict(weight=123, blob='HHHHHHH'), - ]), - ('zone3', [ - dict(weight=130, blob='IIIIIII'), - dict(weight=131, blob='JJJJJJJ'), - dict(weight=132, blob='KKKKKKK'), - dict(weight=3, blob='LLLLLLL'), - ]), - ] - - -class ZoneAwareSchedulerTestCase(test.TestCase): - """Test case for Zone Aware Scheduler.""" - - def test_zone_aware_scheduler(self): - """ - Create a nested set of FakeZones, ensure that a select call returns the - appropriate build plan. - """ - sched = FakeZoneAwareScheduler() - self.stubs.Set(sched, '_call_zone_method', fake_call_zone_method) - - zm = FakeZoneManager() - sched.set_zone_manager(zm) - - fake_context = {} - build_plan = sched.select(fake_context, {}) - - self.assertEqual(15, len(build_plan)) - - hostnames = [plan_item['name'] - for plan_item in build_plan if 'name' in plan_item] - self.assertEqual(3, len(hostnames)) - - def test_empty_zone_aware_scheduler(self): - """ - Ensure empty hosts & child_zones result in NoValidHosts exception. - """ - sched = FakeZoneAwareScheduler() - self.stubs.Set(sched, '_call_zone_method', fake_empty_call_zone_method) - - zm = FakeEmptyZoneManager() - sched.set_zone_manager(zm) - - fake_context = {} - self.assertRaises(driver.NoValidHost, sched.schedule, fake_context, {}) diff --git a/nova/tests/vmwareapi/db_fakes.py b/nova/tests/vmwareapi/db_fakes.py index 0addd5573..d4eb87daf 100644 --- a/nova/tests/vmwareapi/db_fakes.py +++ b/nova/tests/vmwareapi/db_fakes.py @@ -52,7 +52,7 @@ def stub_out_db_instance_api(stubs): else:
raise NotImplementedError()
- def fake_instance_create(values):
+ def fake_instance_create(context, values):
"""Stubs out the db.instance_create method."""
type_data = INSTANCE_TYPES[values['instance_type']]
@@ -61,7 +61,7 @@ def stub_out_db_instance_api(stubs): 'name': values['name'],
'id': values['id'],
'reservation_id': utils.generate_uid('r'),
- 'image_id': values['image_id'],
+ 'image_ref': values['image_ref'],
'kernel_id': values['kernel_id'],
'ramdisk_id': values['ramdisk_id'],
'state_description': 'scheduling',
diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 4833ccb07..151a3e909 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -17,6 +17,7 @@ """Stubouts, mocks and fixtures for the test suite""" import eventlet +import json from nova.virt import xenapi_conn from nova.virt.xenapi import fake from nova.virt.xenapi import volume_utils @@ -37,21 +38,7 @@ def stubout_instance_snapshot(stubs): sr_ref=sr_ref, sharable=False) vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref) vdi_uuid = vdi_rec['uuid'] - return vdi_uuid - - stubs.Set(vm_utils.VMHelper, 'fetch_image', fake_fetch_image) - - def fake_wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref, - original_parent_uuid): - from nova.virt.xenapi.fake import create_vdi - name_label = "instance-%s" % instance_id - #TODO: create fake SR record - sr_ref = "fakesr" - vdi_ref = create_vdi(name_label=name_label, read_only=False, - sr_ref=sr_ref, sharable=False) - vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref) - vdi_uuid = vdi_rec['uuid'] - return vdi_uuid + return [dict(vdi_type='os', vdi_uuid=vdi_uuid)] stubs.Set(vm_utils.VMHelper, 'fetch_image', fake_fetch_image) @@ -132,11 +119,30 @@ class FakeSessionForVMTests(fake.SessionBase): def __init__(self, uri): super(FakeSessionForVMTests, self).__init__(uri) - def host_call_plugin(self, _1, _2, _3, _4, _5): + def host_call_plugin(self, _1, _2, plugin, method, _5): + sr_ref = fake.get_all('SR')[0] + vdi_ref = fake.create_vdi('', False, sr_ref, False) + vdi_rec = fake.get_record('VDI', vdi_ref) + if plugin == "glance" and method == "download_vhd": + ret_str = json.dumps([dict(vdi_type='os', + vdi_uuid=vdi_rec['uuid'])]) + else: + ret_str = vdi_rec['uuid'] + return '<string>%s</string>' % ret_str + + def host_call_plugin_swap(self, _1, _2, plugin, method, _5): sr_ref = fake.get_all('SR')[0] vdi_ref = fake.create_vdi('', False, sr_ref, False) vdi_rec = fake.get_record('VDI', vdi_ref) - return '<string>%s</string>' % vdi_rec['uuid'] + if plugin == "glance" and method == "download_vhd": + swap_vdi_ref = fake.create_vdi('', False, sr_ref, False) + swap_vdi_rec = fake.get_record('VDI', swap_vdi_ref) + ret_str = json.dumps( + [dict(vdi_type='os', vdi_uuid=vdi_rec['uuid']), + dict(vdi_type='swap', vdi_uuid=swap_vdi_rec['uuid'])]) + else: + ret_str = vdi_rec['uuid'] + return '<string>%s</string>' % ret_str def VM_start(self, _1, ref, _2, _3): vm = fake.get_record('VM', ref) @@ -231,10 +237,10 @@ class FakeSessionForMigrationTests(fake.SessionBase): def __init__(self, uri): super(FakeSessionForMigrationTests, self).__init__(uri) - def VDI_get_by_uuid(*args): + def VDI_get_by_uuid(self, *args): return 'hurr' - def VDI_resize_online(*args): + def VDI_resize_online(self, *args): pass def VM_start(self, _1, ref, _2, _3): diff --git a/nova/twistd.py b/nova/twistd.py index c07ed991f..15cf67825 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -78,7 +78,7 @@ def WrapTwistedOptions(wrapped): self._absorbParameters() self._absorbHandlers() - super(TwistedOptionsToFlags, self).__init__() + wrapped.__init__(self) def _absorbFlags(self): twistd_flags = [] @@ -163,12 +163,12 @@ def WrapTwistedOptions(wrapped): def parseArgs(self, *args): # TODO(termie): figure out a decent way of dealing with args #return - super(TwistedOptionsToFlags, self).parseArgs(*args) + wrapped.parseArgs(self, *args) def postOptions(self): self._doHandlers() - super(TwistedOptionsToFlags, self).postOptions() + wrapped.postOptions(self) def __getitem__(self, key): key = key.replace('-', '_') diff --git a/nova/utils.py b/nova/utils.py index 361fc9873..6d8324e5b 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -35,6 +35,7 @@ import struct import sys import time import types +import uuid from xml.sax import saxutils from eventlet import event @@ -142,24 +143,26 @@ def execute(*cmd, **kwargs): env = os.environ.copy() if addl_env: env.update(addl_env) + _PIPE = subprocess.PIPE # pylint: disable=E1101 obj = subprocess.Popen(cmd, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + stdin=_PIPE, + stdout=_PIPE, + stderr=_PIPE, env=env) result = None if process_input is not None: result = obj.communicate(process_input) else: result = obj.communicate() - obj.stdin.close() - if obj.returncode: - LOG.debug(_('Result was %s') % obj.returncode) + obj.stdin.close() # pylint: disable=E1101 + _returncode = obj.returncode # pylint: disable=E1101 + if _returncode: + LOG.debug(_('Result was %s') % _returncode) if type(check_exit_code) == types.IntType \ - and obj.returncode != check_exit_code: + and _returncode != check_exit_code: (stdout, stderr) = result raise exception.ProcessExecutionError( - exit_code=obj.returncode, + exit_code=_returncode, stdout=stdout, stderr=stderr, cmd=' '.join(cmd)) @@ -307,7 +310,7 @@ def get_my_linklocal(interface): def utcnow(): - """Overridable version of datetime.datetime.utcnow.""" + """Overridable version of utils.utcnow.""" if utcnow.override_time: return utcnow.override_time return datetime.datetime.utcnow() @@ -523,6 +526,16 @@ def loads(s): return json.loads(s) +try: + import anyjson +except ImportError: + pass +else: + anyjson._modules.append(("nova.utils", "dumps", TypeError, + "loads", ValueError)) + anyjson.force_implementation("nova.utils") + + _semaphores = {} @@ -724,3 +737,17 @@ def parse_server_string(server_str): except: LOG.debug(_('Invalid server_string: %s' % server_str)) return ('', '') + + +def gen_uuid(): + return uuid.uuid4() + + +def is_uuid_like(val): + """For our purposes, a UUID is a string in canoical form: + + aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa + """ + if not isinstance(val, basestring): + return False + return (len(val) == 36) and (val.count('-') == 4) diff --git a/nova/virt/connection.py b/nova/virt/connection.py index 99a8849f1..aeec17c98 100644 --- a/nova/virt/connection.py +++ b/nova/virt/connection.py @@ -27,9 +27,9 @@ from nova import utils from nova.virt import driver from nova.virt import fake from nova.virt import hyperv -from nova.virt import libvirt_conn from nova.virt import vmwareapi_conn from nova.virt import xenapi_conn +from nova.virt.libvirt import connection as libvirt_conn LOG = logging.getLogger("nova.virt.connection") diff --git a/nova/virt/driver.py b/nova/virt/driver.py index eb9626d08..2c7c0cfcc 100644 --- a/nova/virt/driver.py +++ b/nova/virt/driver.py @@ -61,7 +61,7 @@ class ComputeDriver(object): """Return a list of InstanceInfo for all registered VMs""" raise NotImplementedError() - def spawn(self, instance, network_info=None): + def spawn(self, instance, network_info=None, block_device_mapping=None): """Launch a VM for the specified instance""" raise NotImplementedError() @@ -191,6 +191,10 @@ class ComputeDriver(object): def refresh_security_group_members(self, security_group_id): raise NotImplementedError() + def refresh_provider_fw_rules(self, security_group_id): + """See: nova/virt/fake.py for docs.""" + raise NotImplementedError() + def reset_network(self, instance): """reset networking for specified instance""" raise NotImplementedError() @@ -234,6 +238,10 @@ class ComputeDriver(object): """ raise NotImplementedError() + def agent_update(self, instance, url, md5hash): + """Update agent on the VM instance.""" + raise NotImplementedError() + def inject_network_info(self, instance): """inject network info for specified instance""" raise NotImplementedError() diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 5ac376e46..f78c29bd0 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -82,6 +82,21 @@ class FakeConnection(driver.ComputeDriver): def __init__(self): self.instances = {} + self.host_status = { + 'host_name-description': 'Fake Host', + 'host_hostname': 'fake-mini', + 'host_memory_total': 8000000000, + 'host_memory_overhead': 10000000, + 'host_memory_free': 7900000000, + 'host_memory_free_computed': 7900000000, + 'host_other_config': {}, + 'host_ip_address': '192.168.1.109', + 'host_cpu_info': {}, + 'disk_available': 500000000000, + 'disk_total': 600000000000, + 'disk_used': 100000000000, + 'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f', + 'host_name_label': 'fake-mini'} @classmethod def instance(cls): @@ -114,7 +129,7 @@ class FakeConnection(driver.ComputeDriver): info_list.append(self._map_to_instance_info(instance)) return info_list - def spawn(self, instance): + def spawn(self, instance, network_info=None, block_device_mapping=None): """ Create a new instance/VM/domain on the virtualization platform. @@ -210,6 +225,21 @@ class FakeConnection(driver.ComputeDriver): """ pass + def agent_update(self, instance, url, md5hash): + """ + Update agent on the specified instance. + + The first parameter is an instance of nova.compute.service.Instance, + and so the instance is being specified as instance.name. The second + parameter is the URL of the agent to be fetched and updated on the + instance; the third is the md5 hash of the file for verification + purposes. + + The work will be done asynchronously. This function returns a + task that allows the caller to detect when it is complete. + """ + pass + def rescue(self, instance): """ Rescue the specified instance. @@ -222,6 +252,10 @@ class FakeConnection(driver.ComputeDriver): """ pass + def poll_rescued_instances(self, timeout): + """Poll for rescued instances""" + pass + def migrate_disk_and_power_off(self, instance, dest): """ Transfers the disk of a running instance in multiple phases, turning @@ -432,6 +466,22 @@ class FakeConnection(driver.ComputeDriver): """ return True + def refresh_provider_fw_rules(self): + """This triggers a firewall update based on database changes. + + When this is called, rules have either been added or removed from the + datastore. You can retrieve rules with + :method:`nova.db.api.provider_fw_rule_get_all`. + + Provider rules take precedence over security group rules. If an IP + would be allowed by a security group ingress rule, but blocked by + a provider rule, then packets from the IP are dropped. This includes + intra-project traffic in the case of the allow_project_net_traffic + flag for the libvirt-derived classes. + + """ + pass + def update_available_resource(self, ctxt, host): """This method is supported only by libvirt.""" return @@ -456,3 +506,11 @@ class FakeConnection(driver.ComputeDriver): def test_remove_vm(self, instance_name): """ Removes the named VM, as if it crashed. For testing""" self.instances.pop(instance_name) + + def update_host_status(self): + """Return fake Host Status of ram, disk, network.""" + return self.host_status + + def get_host_stats(self, refresh=False): + """Return fake Host Status of ram, disk, network.""" + return self.host_status diff --git a/nova/virt/hyperv.py b/nova/virt/hyperv.py index 1142e97a4..772e7eb59 100644 --- a/nova/virt/hyperv.py +++ b/nova/virt/hyperv.py @@ -139,7 +139,7 @@ class HyperVConnection(driver.ComputeDriver): return instance_infos - def spawn(self, instance): + def spawn(self, instance, network_info=None, block_device_mapping=None): """ Create a new VM and start it.""" vm = self._lookup(instance.name) if vm is not None: @@ -151,7 +151,7 @@ class HyperVConnection(driver.ComputeDriver): base_vhd_filename = os.path.join(FLAGS.instances_path, instance.name) vhdfile = "%s.vhd" % (base_vhd_filename) - images.fetch(instance['image_id'], vhdfile, user, project) + images.fetch(instance['image_ref'], vhdfile, user, project) try: self._create_vm(instance) diff --git a/nova/virt/images.py b/nova/virt/images.py index 02c898fda..40bf6107c 100644 --- a/nova/virt/images.py +++ b/nova/virt/images.py @@ -23,6 +23,8 @@ Handling of VM disk images. from nova import context from nova import flags +from nova.image import glance as glance_image_service +import nova.image from nova import log as logging from nova import utils @@ -31,23 +33,13 @@ FLAGS = flags.FLAGS LOG = logging.getLogger('nova.virt.images') -def fetch(image_id, path, _user, _project): +def fetch(image_href, path, _user, _project): # TODO(vish): Improve context handling and add owner and auth data # when it is added to glance. Right now there is no # auth checking in glance, so we assume that access was # checked before we got here. - image_service = utils.import_object(FLAGS.image_service) + (image_service, image_id) = nova.image.get_image_service(image_href) with open(path, "wb") as image_file: elevated = context.get_admin_context() metadata = image_service.get(elevated, image_id, image_file) return metadata - - -# TODO(vish): xenapi should use the glance client code directly instead -# of retrieving the image using this method. -def image_url(image): - if FLAGS.image_service == "nova.image.glance.GlanceImageService": - return "http://%s:%s/images/%s" % (FLAGS.glance_host, - FLAGS.glance_port, image) - return "http://%s:%s/_images/%s/image" % (FLAGS.s3_host, FLAGS.s3_port, - image) diff --git a/nova/virt/libvirt.xml.template b/nova/virt/libvirt.xml.template index de2497a76..e1a683da8 100644 --- a/nova/virt/libvirt.xml.template +++ b/nova/virt/libvirt.xml.template @@ -67,11 +67,13 @@ <target dev='${disk_prefix}b' bus='${disk_bus}'/> </disk> #else + #if not ($getVar('ebs_root', False)) <disk type='file'> <driver type='${driver_type}'/> <source file='${basepath}/disk'/> <target dev='${disk_prefix}a' bus='${disk_bus}'/> </disk> + #end if #if $getVar('local', False) <disk type='file'> <driver type='${driver_type}'/> @@ -79,6 +81,13 @@ <target dev='${disk_prefix}b' bus='${disk_bus}'/> </disk> #end if + #for $vol in $volumes + <disk type='block'> + <driver type='raw'/> + <source dev='${vol.device_path}'/> + <target dev='${vol.mount_device}' bus='${disk_bus}'/> + </disk> + #end for #end if #end if @@ -116,7 +125,7 @@ </serial> #if $getVar('vncserver_host', False) - <graphics type='vnc' port='-1' autoport='yes' keymap='en-us' listen='${vncserver_host}'/> + <graphics type='vnc' port='-1' autoport='yes' keymap='${vnc_keymap}' listen='${vncserver_host}'/> #end if </devices> </domain> diff --git a/nova/virt/libvirt/__init__.py b/nova/virt/libvirt/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/nova/virt/libvirt/__init__.py diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt/connection.py index fa918b0a3..b06bfb714 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt/connection.py @@ -36,9 +36,12 @@ Supports KVM, LXC, QEMU, UML, and XEN. """ +import hashlib import multiprocessing +import netaddr import os import random +import re import shutil import subprocess import sys @@ -51,13 +54,11 @@ from xml.etree import ElementTree from eventlet import greenthread from eventlet import tpool -import IPy - from nova import context from nova import db from nova import exception from nova import flags -from nova import ipv6 +import nova.image from nova import log as logging from nova import utils from nova import vnc @@ -67,20 +68,23 @@ from nova.compute import power_state from nova.virt import disk from nova.virt import driver from nova.virt import images +from nova.virt.libvirt import netutils + libvirt = None libxml2 = None Template = None + LOG = logging.getLogger('nova.virt.libvirt_conn') + FLAGS = flags.FLAGS flags.DECLARE('live_migration_retry_count', 'nova.compute.manager') # TODO(vish): These flags should probably go into a shared location flags.DEFINE_string('rescue_image_id', 'ami-rescue', 'Rescue ami image') flags.DEFINE_string('rescue_kernel_id', 'aki-rescue', 'Rescue aki image') flags.DEFINE_string('rescue_ramdisk_id', 'ari-rescue', 'Rescue ari image') - flags.DEFINE_string('libvirt_xml_template', utils.abspath('virt/libvirt.xml.template'), 'Libvirt XML Template') @@ -102,7 +106,7 @@ flags.DEFINE_string('ajaxterm_portrange', '10000-12000', 'Range of ports that ajaxterm should randomly try to bind') flags.DEFINE_string('firewall_driver', - 'nova.virt.libvirt_conn.IptablesFirewallDriver', + 'nova.virt.libvirt.firewall.IptablesFirewallDriver', 'Firewall driver (defaults to iptables)') flags.DEFINE_string('cpuinfo_xml_template', utils.abspath('virt/cpuinfo.xml.template'), @@ -144,68 +148,8 @@ def _late_load_cheetah(): Template = t.Template -def _get_net_and_mask(cidr): - net = IPy.IP(cidr) - return str(net.net()), str(net.netmask()) - - -def _get_net_and_prefixlen(cidr): - net = IPy.IP(cidr) - return str(net.net()), str(net.prefixlen()) - - -def _get_ip_version(cidr): - net = IPy.IP(cidr) - return int(net.version()) - - -def _get_network_info(instance): - # TODO(adiantum) If we will keep this function - # we should cache network_info - admin_context = context.get_admin_context() - - ip_addresses = db.fixed_ip_get_all_by_instance(admin_context, - instance['id']) - networks = db.network_get_all_by_instance(admin_context, - instance['id']) - flavor = db.instance_type_get_by_id(admin_context, - instance['instance_type_id']) - network_info = [] - - for network in networks: - network_ips = [ip for ip in ip_addresses - if ip['network_id'] == network['id']] - - def ip_dict(ip): - return { - 'ip': ip['address'], - 'netmask': network['netmask'], - 'enabled': '1'} - - def ip6_dict(): - prefix = network['cidr_v6'] - mac = instance['mac_address'] - project_id = instance['project_id'] - return { - 'ip': ipv6.to_global(prefix, mac, project_id), - 'netmask': network['netmask_v6'], - 'enabled': '1'} - - mapping = { - 'label': network['label'], - 'gateway': network['gateway'], - 'broadcast': network['broadcast'], - 'mac': instance['mac_address'], - 'rxtx_cap': flavor['rxtx_cap'], - 'dns': [network['dns']], - 'ips': [ip_dict(ip) for ip in network_ips]} - - if FLAGS.use_ipv6: - mapping['ip6s'] = [ip6_dict()] - mapping['gateway6'] = network['gateway_v6'] - - network_info.append((network, mapping)) - return network_info +def _strip_dev(mount_path): + return re.sub(r'^/dev/', '', mount_path) class LibvirtConnection(driver.ComputeDriver): @@ -440,7 +384,7 @@ class LibvirtConnection(driver.ComputeDriver): virt_dom.detachDevice(xml) @exception.wrap_exception - def snapshot(self, instance, image_id): + def snapshot(self, instance, image_href): """Create snapshot from a running VM instance. This command only works with qemu 0.14+, the qemu_img flag is @@ -448,17 +392,22 @@ class LibvirtConnection(driver.ComputeDriver): to support this command. """ - image_service = utils.import_object(FLAGS.image_service) virt_dom = self._lookup_by_name(instance['name']) elevated = context.get_admin_context() - base = image_service.show(elevated, instance['image_id']) + (image_service, image_id) = nova.image.get_image_service( + instance['image_ref']) + base = image_service.show(elevated, image_id) + (snapshot_image_service, snapshot_image_id) = \ + nova.image.get_image_service(image_href) + snapshot = snapshot_image_service.show(elevated, snapshot_image_id) metadata = {'disk_format': base['disk_format'], 'container_format': base['container_format'], 'is_public': False, - 'name': '%s.%s' % (base['name'], image_id), - 'properties': {'architecture': base['architecture'], + 'status': 'active', + 'name': snapshot['name'], + 'properties': { 'kernel_id': instance['kernel_id'], 'image_location': 'snapshot', 'image_state': 'available', @@ -466,6 +415,9 @@ class LibvirtConnection(driver.ComputeDriver): 'ramdisk_id': instance['ramdisk_id'], } } + if 'architecture' in base['properties']: + arch = base['properties']['architecture'] + metadata['properties']['architecture'] = arch # Make the snapshot snapshot_name = uuid.uuid4().hex @@ -500,7 +452,7 @@ class LibvirtConnection(driver.ComputeDriver): # Upload that image to the image service with open(out_path) as image_file: image_service.update(elevated, - image_id, + image_href, metadata, image_file) @@ -550,19 +502,27 @@ class LibvirtConnection(driver.ComputeDriver): @exception.wrap_exception def pause(self, instance, callback): - raise exception.ApiError("pause not supported for libvirt.") + """Pause VM instance""" + dom = self._lookup_by_name(instance.name) + dom.suspend() @exception.wrap_exception def unpause(self, instance, callback): - raise exception.ApiError("unpause not supported for libvirt.") + """Unpause paused VM instance""" + dom = self._lookup_by_name(instance.name) + dom.resume() @exception.wrap_exception def suspend(self, instance, callback): - raise exception.ApiError("suspend not supported for libvirt") + """Suspend the specified instance""" + dom = self._lookup_by_name(instance.name) + dom.managedSave(0) @exception.wrap_exception def resume(self, instance, callback): - raise exception.ApiError("resume not supported for libvirt") + """resume the specified instance""" + dom = self._lookup_by_name(instance.name) + dom.create() @exception.wrap_exception def rescue(self, instance): @@ -619,11 +579,14 @@ class LibvirtConnection(driver.ComputeDriver): # NOTE(ilyaalekseyev): Implementation like in multinics # for xenapi(tr3buchet) @exception.wrap_exception - def spawn(self, instance, network_info=None): - xml = self.to_xml(instance, False, network_info) + def spawn(self, instance, network_info=None, block_device_mapping=None): + xml = self.to_xml(instance, False, network_info=network_info, + block_device_mapping=block_device_mapping) + block_device_mapping = block_device_mapping or [] self.firewall_driver.setup_basic_filtering(instance, network_info) self.firewall_driver.prepare_instance_filter(instance, network_info) - self._create_image(instance, xml, network_info=network_info) + self._create_image(instance, xml, network_info=network_info, + block_device_mapping=block_device_mapping) domain = self._create_new_domain(xml) LOG.debug(_("instance %s: is running"), instance['name']) self.firewall_driver.apply_instance_filter(instance) @@ -805,9 +768,10 @@ class LibvirtConnection(driver.ComputeDriver): # TODO(vish): should we format disk by default? def _create_image(self, inst, libvirt_xml, suffix='', disk_images=None, - network_info=None): + network_info=None, block_device_mapping=None): + block_device_mapping = block_device_mapping or [] if not network_info: - network_info = _get_network_info(inst) + network_info = netutils.get_network_info(inst) if not suffix: suffix = '' @@ -838,7 +802,7 @@ class LibvirtConnection(driver.ComputeDriver): project = manager.AuthManager().get_project(inst['project_id']) if not disk_images: - disk_images = {'image_id': inst['image_id'], + disk_images = {'image_id': inst['image_ref'], 'kernel_id': inst['kernel_id'], 'ramdisk_id': inst['ramdisk_id']} @@ -859,7 +823,7 @@ class LibvirtConnection(driver.ComputeDriver): user=user, project=project) - root_fname = '%08x' % int(disk_images['image_id']) + root_fname = hashlib.sha1(disk_images['image_id']).hexdigest() size = FLAGS.minimum_root_size inst_type_id = inst['instance_type_id'] @@ -868,16 +832,19 @@ class LibvirtConnection(driver.ComputeDriver): size = None root_fname += "_sm" - self._cache_image(fn=self._fetch_image, - target=basepath('disk'), - fname=root_fname, - cow=FLAGS.use_cow_images, - image_id=disk_images['image_id'], - user=user, - project=project, - size=size) + if not self._volume_in_mapping(self.root_mount_device, + block_device_mapping): + self._cache_image(fn=self._fetch_image, + target=basepath('disk'), + fname=root_fname, + cow=FLAGS.use_cow_images, + image_id=disk_images['image_id'], + user=user, + project=project, + size=size) - if inst_type['local_gb']: + if inst_type['local_gb'] and not self._volume_in_mapping( + self.local_mount_device, block_device_mapping): self._cache_image(fn=self._create_local, target=basepath('disk.local'), fname="local_%s" % inst_type['local_gb'], @@ -934,7 +901,7 @@ class LibvirtConnection(driver.ComputeDriver): if key or net: inst_name = inst['name'] - img_id = inst.image_id + img_id = inst.image_ref if key: LOG.info(_('instance %(inst_name)s: injecting key into' ' image %(img_id)s') % locals()) @@ -966,10 +933,10 @@ class LibvirtConnection(driver.ComputeDriver): if FLAGS.allow_project_net_traffic: template = "<parameter name=\"%s\"value=\"%s\" />\n" - net, mask = _get_net_and_mask(network['cidr']) + net, mask = netutils.get_net_and_mask(network['cidr']) values = [("PROJNET", net), ("PROJMASK", mask)] if FLAGS.use_ipv6: - net_v6, prefixlen_v6 = _get_net_and_prefixlen( + net_v6, prefixlen_v6 = netutils.get_net_and_prefixlen( network['cidr_v6']) values.extend([("PROJNETV6", net_v6), ("PROJMASKV6", prefixlen_v6)]) @@ -992,11 +959,24 @@ class LibvirtConnection(driver.ComputeDriver): return result - def _prepare_xml_info(self, instance, rescue=False, network_info=None): + root_mount_device = 'vda' # FIXME for now. it's hard coded. + local_mount_device = 'vdb' # FIXME for now. it's hard coded. + + def _volume_in_mapping(self, mount_device, block_device_mapping): + mount_device_ = _strip_dev(mount_device) + for vol in block_device_mapping: + vol_mount_device = _strip_dev(vol['mount_device']) + if vol_mount_device == mount_device_: + return True + return False + + def _prepare_xml_info(self, instance, rescue=False, network_info=None, + block_device_mapping=None): + block_device_mapping = block_device_mapping or [] # TODO(adiantum) remove network_info creation code # when multinics will be completed if not network_info: - network_info = _get_network_info(instance) + network_info = netutils.get_network_info(instance) nics = [] for (network, mapping) in network_info: @@ -1010,6 +990,16 @@ class LibvirtConnection(driver.ComputeDriver): else: driver_type = 'raw' + for vol in block_device_mapping: + vol['mount_device'] = _strip_dev(vol['mount_device']) + ebs_root = self._volume_in_mapping(self.root_mount_device, + block_device_mapping) + if self._volume_in_mapping(self.local_mount_device, + block_device_mapping): + local_gb = False + else: + local_gb = inst_type['local_gb'] + xml_info = {'type': FLAGS.libvirt_type, 'name': instance['name'], 'basepath': os.path.join(FLAGS.instances_path, @@ -1017,13 +1007,16 @@ class LibvirtConnection(driver.ComputeDriver): 'memory_kb': inst_type['memory_mb'] * 1024, 'vcpus': inst_type['vcpus'], 'rescue': rescue, - 'local': inst_type['local_gb'], + 'local': local_gb, 'driver_type': driver_type, - 'nics': nics} + 'nics': nics, + 'ebs_root': ebs_root, + 'volumes': block_device_mapping} if FLAGS.vnc_enabled: if FLAGS.libvirt_type != 'lxc': xml_info['vncserver_host'] = FLAGS.vncserver_host + xml_info['vnc_keymap'] = FLAGS.vnc_keymap if not rescue: if instance['kernel_id']: xml_info['kernel'] = xml_info['basepath'] + "/kernel" @@ -1034,10 +1027,13 @@ class LibvirtConnection(driver.ComputeDriver): xml_info['disk'] = xml_info['basepath'] + "/disk" return xml_info - def to_xml(self, instance, rescue=False, network_info=None): + def to_xml(self, instance, rescue=False, network_info=None, + block_device_mapping=None): + block_device_mapping = block_device_mapping or [] # TODO(termie): cache? LOG.debug(_('instance %s: starting toXML method'), instance['name']) - xml_info = self._prepare_xml_info(instance, rescue, network_info) + xml_info = self._prepare_xml_info(instance, rescue, network_info, + block_device_mapping) xml = str(Template(self.libvirt_xml, searchList=[xml_info])) LOG.debug(_('instance %s: finished toXML method'), instance['name']) return xml @@ -1386,6 +1382,9 @@ class LibvirtConnection(driver.ComputeDriver): def refresh_security_group_members(self, security_group_id): self.firewall_driver.refresh_security_group_members(security_group_id) + def refresh_provider_fw_rules(self): + self.firewall_driver.refresh_provider_fw_rules() + def update_available_resource(self, ctxt, host): """Updates compute manager resource info on ComputeNode table. @@ -1591,606 +1590,3 @@ class LibvirtConnection(driver.ComputeDriver): def get_host_stats(self, refresh=False): """See xenapi_conn.py implementation.""" pass - - -class FirewallDriver(object): - def prepare_instance_filter(self, instance, network_info=None): - """Prepare filters for the instance. - - At this point, the instance isn't running yet.""" - raise NotImplementedError() - - def unfilter_instance(self, instance): - """Stop filtering instance""" - raise NotImplementedError() - - def apply_instance_filter(self, instance): - """Apply instance filter. - - Once this method returns, the instance should be firewalled - appropriately. This method should as far as possible be a - no-op. It's vastly preferred to get everything set up in - prepare_instance_filter. - """ - raise NotImplementedError() - - def refresh_security_group_rules(self, - security_group_id, - network_info=None): - """Refresh security group rules from data store - - Gets called when a rule has been added to or removed from - the security group.""" - raise NotImplementedError() - - def refresh_security_group_members(self, security_group_id): - """Refresh security group members from data store - - Gets called when an instance gets added to or removed from - the security group.""" - raise NotImplementedError() - - def setup_basic_filtering(self, instance, network_info=None): - """Create rules to block spoofing and allow dhcp. - - This gets called when spawning an instance, before - :method:`prepare_instance_filter`. - - """ - raise NotImplementedError() - - def instance_filter_exists(self, instance): - """Check nova-instance-instance-xxx exists""" - raise NotImplementedError() - - -class NWFilterFirewall(FirewallDriver): - """ - This class implements a network filtering mechanism versatile - enough for EC2 style Security Group filtering by leveraging - libvirt's nwfilter. - - First, all instances get a filter ("nova-base-filter") applied. - This filter provides some basic security such as protection against - MAC spoofing, IP spoofing, and ARP spoofing. - - This filter drops all incoming ipv4 and ipv6 connections. - Outgoing connections are never blocked. - - Second, every security group maps to a nwfilter filter(*). - NWFilters can be updated at runtime and changes are applied - immediately, so changes to security groups can be applied at - runtime (as mandated by the spec). - - Security group rules are named "nova-secgroup-<id>" where <id> - is the internal id of the security group. They're applied only on - hosts that have instances in the security group in question. - - Updates to security groups are done by updating the data model - (in response to API calls) followed by a request sent to all - the nodes with instances in the security group to refresh the - security group. - - Each instance has its own NWFilter, which references the above - mentioned security group NWFilters. This was done because - interfaces can only reference one filter while filters can - reference multiple other filters. This has the added benefit of - actually being able to add and remove security groups from an - instance at run time. This functionality is not exposed anywhere, - though. - - Outstanding questions: - - The name is unique, so would there be any good reason to sync - the uuid across the nodes (by assigning it from the datamodel)? - - - (*) This sentence brought to you by the redundancy department of - redundancy. - - """ - - def __init__(self, get_connection, **kwargs): - self._libvirt_get_connection = get_connection - self.static_filters_configured = False - self.handle_security_groups = False - - def apply_instance_filter(self, instance): - """No-op. Everything is done in prepare_instance_filter""" - pass - - def _get_connection(self): - return self._libvirt_get_connection() - _conn = property(_get_connection) - - def nova_dhcp_filter(self): - """The standard allow-dhcp-server filter is an <ip> one, so it uses - ebtables to allow traffic through. Without a corresponding rule in - iptables, it'll get blocked anyway.""" - - return '''<filter name='nova-allow-dhcp-server' chain='ipv4'> - <uuid>891e4787-e5c0-d59b-cbd6-41bc3c6b36fc</uuid> - <rule action='accept' direction='out' - priority='100'> - <udp srcipaddr='0.0.0.0' - dstipaddr='255.255.255.255' - srcportstart='68' - dstportstart='67'/> - </rule> - <rule action='accept' direction='in' - priority='100'> - <udp srcipaddr='$DHCPSERVER' - srcportstart='67' - dstportstart='68'/> - </rule> - </filter>''' - - def nova_ra_filter(self): - return '''<filter name='nova-allow-ra-server' chain='root'> - <uuid>d707fa71-4fb5-4b27-9ab7-ba5ca19c8804</uuid> - <rule action='accept' direction='inout' - priority='100'> - <icmpv6 srcipaddr='$RASERVER'/> - </rule> - </filter>''' - - def setup_basic_filtering(self, instance, network_info=None): - """Set up basic filtering (MAC, IP, and ARP spoofing protection)""" - logging.info('called setup_basic_filtering in nwfilter') - - if not network_info: - network_info = _get_network_info(instance) - - if self.handle_security_groups: - # No point in setting up a filter set that we'll be overriding - # anyway. - return - - logging.info('ensuring static filters') - self._ensure_static_filters() - - if instance['image_id'] == str(FLAGS.vpn_image_id): - base_filter = 'nova-vpn' - else: - base_filter = 'nova-base' - - for (network, mapping) in network_info: - nic_id = mapping['mac'].replace(':', '') - instance_filter_name = self._instance_filter_name(instance, nic_id) - self._define_filter(self._filter_container(instance_filter_name, - [base_filter])) - - def _ensure_static_filters(self): - if self.static_filters_configured: - return - - self._define_filter(self._filter_container('nova-base', - ['no-mac-spoofing', - 'no-ip-spoofing', - 'no-arp-spoofing', - 'allow-dhcp-server'])) - self._define_filter(self._filter_container('nova-vpn', - ['allow-dhcp-server'])) - self._define_filter(self.nova_base_ipv4_filter) - self._define_filter(self.nova_base_ipv6_filter) - self._define_filter(self.nova_dhcp_filter) - self._define_filter(self.nova_ra_filter) - if FLAGS.allow_project_net_traffic: - self._define_filter(self.nova_project_filter) - if FLAGS.use_ipv6: - self._define_filter(self.nova_project_filter_v6) - - self.static_filters_configured = True - - def _filter_container(self, name, filters): - xml = '''<filter name='%s' chain='root'>%s</filter>''' % ( - name, - ''.join(["<filterref filter='%s'/>" % (f,) for f in filters])) - return xml - - def nova_base_ipv4_filter(self): - retval = "<filter name='nova-base-ipv4' chain='ipv4'>" - for protocol in ['tcp', 'udp', 'icmp']: - for direction, action, priority in [('out', 'accept', 399), - ('in', 'drop', 400)]: - retval += """<rule action='%s' direction='%s' priority='%d'> - <%s /> - </rule>""" % (action, direction, - priority, protocol) - retval += '</filter>' - return retval - - def nova_base_ipv6_filter(self): - retval = "<filter name='nova-base-ipv6' chain='ipv6'>" - for protocol in ['tcp-ipv6', 'udp-ipv6', 'icmpv6']: - for direction, action, priority in [('out', 'accept', 399), - ('in', 'drop', 400)]: - retval += """<rule action='%s' direction='%s' priority='%d'> - <%s /> - </rule>""" % (action, direction, - priority, protocol) - retval += '</filter>' - return retval - - def nova_project_filter(self): - retval = "<filter name='nova-project' chain='ipv4'>" - for protocol in ['tcp', 'udp', 'icmp']: - retval += """<rule action='accept' direction='in' priority='200'> - <%s srcipaddr='$PROJNET' srcipmask='$PROJMASK' /> - </rule>""" % protocol - retval += '</filter>' - return retval - - def nova_project_filter_v6(self): - retval = "<filter name='nova-project-v6' chain='ipv6'>" - for protocol in ['tcp-ipv6', 'udp-ipv6', 'icmpv6']: - retval += """<rule action='accept' direction='inout' - priority='200'> - <%s srcipaddr='$PROJNETV6' - srcipmask='$PROJMASKV6' /> - </rule>""" % (protocol) - retval += '</filter>' - return retval - - def _define_filter(self, xml): - if callable(xml): - xml = xml() - # execute in a native thread and block current greenthread until done - tpool.execute(self._conn.nwfilterDefineXML, xml) - - def unfilter_instance(self, instance): - # Nothing to do - pass - - def prepare_instance_filter(self, instance, network_info=None): - """ - Creates an NWFilter for the given instance. In the process, - it makes sure the filters for the security groups as well as - the base filter are all in place. - """ - if not network_info: - network_info = _get_network_info(instance) - - ctxt = context.get_admin_context() - - instance_secgroup_filter_name = \ - '%s-secgroup' % (self._instance_filter_name(instance)) - #% (instance_filter_name,) - - instance_secgroup_filter_children = ['nova-base-ipv4', - 'nova-base-ipv6', - 'nova-allow-dhcp-server'] - - if FLAGS.use_ipv6: - networks = [network for (network, _m) in network_info if - network['gateway_v6']] - - if networks: - instance_secgroup_filter_children.\ - append('nova-allow-ra-server') - - for security_group in \ - db.security_group_get_by_instance(ctxt, instance['id']): - - self.refresh_security_group_rules(security_group['id']) - - instance_secgroup_filter_children.append('nova-secgroup-%s' % - security_group['id']) - - self._define_filter( - self._filter_container(instance_secgroup_filter_name, - instance_secgroup_filter_children)) - - network_filters = self.\ - _create_network_filters(instance, network_info, - instance_secgroup_filter_name) - - for (name, children) in network_filters: - self._define_filters(name, children) - - def _create_network_filters(self, instance, network_info, - instance_secgroup_filter_name): - if instance['image_id'] == str(FLAGS.vpn_image_id): - base_filter = 'nova-vpn' - else: - base_filter = 'nova-base' - - result = [] - for (_n, mapping) in network_info: - nic_id = mapping['mac'].replace(':', '') - instance_filter_name = self._instance_filter_name(instance, nic_id) - instance_filter_children = [base_filter, - instance_secgroup_filter_name] - - if FLAGS.allow_project_net_traffic: - instance_filter_children.append('nova-project') - if FLAGS.use_ipv6: - instance_filter_children.append('nova-project-v6') - - result.append((instance_filter_name, instance_filter_children)) - - return result - - def _define_filters(self, filter_name, filter_children): - self._define_filter(self._filter_container(filter_name, - filter_children)) - - def refresh_security_group_rules(self, - security_group_id, - network_info=None): - return self._define_filter( - self.security_group_to_nwfilter_xml(security_group_id)) - - def security_group_to_nwfilter_xml(self, security_group_id): - security_group = db.security_group_get(context.get_admin_context(), - security_group_id) - rule_xml = "" - v6protocol = {'tcp': 'tcp-ipv6', 'udp': 'udp-ipv6', 'icmp': 'icmpv6'} - for rule in security_group.rules: - rule_xml += "<rule action='accept' direction='in' priority='300'>" - if rule.cidr: - version = _get_ip_version(rule.cidr) - if(FLAGS.use_ipv6 and version == 6): - net, prefixlen = _get_net_and_prefixlen(rule.cidr) - rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \ - (v6protocol[rule.protocol], net, prefixlen) - else: - net, mask = _get_net_and_mask(rule.cidr) - rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \ - (rule.protocol, net, mask) - if rule.protocol in ['tcp', 'udp']: - rule_xml += "dstportstart='%s' dstportend='%s' " % \ - (rule.from_port, rule.to_port) - elif rule.protocol == 'icmp': - LOG.info('rule.protocol: %r, rule.from_port: %r, ' - 'rule.to_port: %r', rule.protocol, - rule.from_port, rule.to_port) - if rule.from_port != -1: - rule_xml += "type='%s' " % rule.from_port - if rule.to_port != -1: - rule_xml += "code='%s' " % rule.to_port - - rule_xml += '/>\n' - rule_xml += "</rule>\n" - xml = "<filter name='nova-secgroup-%s' " % security_group_id - if(FLAGS.use_ipv6): - xml += "chain='root'>%s</filter>" % rule_xml - else: - xml += "chain='ipv4'>%s</filter>" % rule_xml - return xml - - def _instance_filter_name(self, instance, nic_id=None): - if not nic_id: - return 'nova-instance-%s' % (instance['name']) - return 'nova-instance-%s-%s' % (instance['name'], nic_id) - - def instance_filter_exists(self, instance): - """Check nova-instance-instance-xxx exists""" - network_info = _get_network_info(instance) - for (network, mapping) in network_info: - nic_id = mapping['mac'].replace(':', '') - instance_filter_name = self._instance_filter_name(instance, nic_id) - try: - self._conn.nwfilterLookupByName(instance_filter_name) - except libvirt.libvirtError: - name = instance.name - LOG.debug(_('The nwfilter(%(instance_filter_name)s) for' - '%(name)s is not found.') % locals()) - return False - return True - - -class IptablesFirewallDriver(FirewallDriver): - def __init__(self, execute=None, **kwargs): - from nova.network import linux_net - self.iptables = linux_net.iptables_manager - self.instances = {} - self.nwfilter = NWFilterFirewall(kwargs['get_connection']) - - self.iptables.ipv4['filter'].add_chain('sg-fallback') - self.iptables.ipv4['filter'].add_rule('sg-fallback', '-j DROP') - self.iptables.ipv6['filter'].add_chain('sg-fallback') - self.iptables.ipv6['filter'].add_rule('sg-fallback', '-j DROP') - - def setup_basic_filtering(self, instance, network_info=None): - """Use NWFilter from libvirt for this.""" - if not network_info: - network_info = _get_network_info(instance) - return self.nwfilter.setup_basic_filtering(instance, network_info) - - def apply_instance_filter(self, instance): - """No-op. Everything is done in prepare_instance_filter""" - pass - - def unfilter_instance(self, instance): - if self.instances.pop(instance['id'], None): - self.remove_filters_for_instance(instance) - self.iptables.apply() - else: - LOG.info(_('Attempted to unfilter instance %s which is not ' - 'filtered'), instance['id']) - - def prepare_instance_filter(self, instance, network_info=None): - if not network_info: - network_info = _get_network_info(instance) - self.instances[instance['id']] = instance - self.add_filters_for_instance(instance, network_info) - self.iptables.apply() - - def _create_filter(self, ips, chain_name): - return ['-d %s -j $%s' % (ip, chain_name) for ip in ips] - - def _filters_for_instance(self, chain_name, network_info): - ips_v4 = [ip['ip'] for (_n, mapping) in network_info - for ip in mapping['ips']] - ipv4_rules = self._create_filter(ips_v4, chain_name) - - ipv6_rules = [] - if FLAGS.use_ipv6: - ips_v6 = [ip['ip'] for (_n, mapping) in network_info - for ip in mapping['ip6s']] - ipv6_rules = self._create_filter(ips_v6, chain_name) - - return ipv4_rules, ipv6_rules - - def _add_filters(self, chain_name, ipv4_rules, ipv6_rules): - for rule in ipv4_rules: - self.iptables.ipv4['filter'].add_rule(chain_name, rule) - - if FLAGS.use_ipv6: - for rule in ipv6_rules: - self.iptables.ipv6['filter'].add_rule(chain_name, rule) - - def add_filters_for_instance(self, instance, network_info=None): - chain_name = self._instance_chain_name(instance) - if FLAGS.use_ipv6: - self.iptables.ipv6['filter'].add_chain(chain_name) - self.iptables.ipv4['filter'].add_chain(chain_name) - ipv4_rules, ipv6_rules = self._filters_for_instance(chain_name, - network_info) - self._add_filters('local', ipv4_rules, ipv6_rules) - ipv4_rules, ipv6_rules = self.instance_rules(instance, network_info) - self._add_filters(chain_name, ipv4_rules, ipv6_rules) - - def remove_filters_for_instance(self, instance): - chain_name = self._instance_chain_name(instance) - - self.iptables.ipv4['filter'].remove_chain(chain_name) - if FLAGS.use_ipv6: - self.iptables.ipv6['filter'].remove_chain(chain_name) - - def instance_rules(self, instance, network_info=None): - if not network_info: - network_info = _get_network_info(instance) - ctxt = context.get_admin_context() - - ipv4_rules = [] - ipv6_rules = [] - - # Always drop invalid packets - ipv4_rules += ['-m state --state ' 'INVALID -j DROP'] - ipv6_rules += ['-m state --state ' 'INVALID -j DROP'] - - # Allow established connections - ipv4_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT'] - ipv6_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT'] - - dhcp_servers = [network['gateway'] for (network, _m) in network_info] - - for dhcp_server in dhcp_servers: - ipv4_rules.append('-s %s -p udp --sport 67 --dport 68 ' - '-j ACCEPT' % (dhcp_server,)) - - #Allow project network traffic - if FLAGS.allow_project_net_traffic: - cidrs = [network['cidr'] for (network, _m) in network_info] - for cidr in cidrs: - ipv4_rules.append('-s %s -j ACCEPT' % (cidr,)) - - # We wrap these in FLAGS.use_ipv6 because they might cause - # a DB lookup. The other ones are just list operations, so - # they're not worth the clutter. - if FLAGS.use_ipv6: - # Allow RA responses - gateways_v6 = [network['gateway_v6'] for (network, _) in - network_info] - for gateway_v6 in gateways_v6: - ipv6_rules.append( - '-s %s/128 -p icmpv6 -j ACCEPT' % (gateway_v6,)) - - #Allow project network traffic - if FLAGS.allow_project_net_traffic: - cidrv6s = [network['cidr_v6'] for (network, _m) - in network_info] - - for cidrv6 in cidrv6s: - ipv6_rules.append('-s %s -j ACCEPT' % (cidrv6,)) - - security_groups = db.security_group_get_by_instance(ctxt, - instance['id']) - - # then, security group chains and rules - for security_group in security_groups: - rules = db.security_group_rule_get_by_security_group(ctxt, - security_group['id']) - - for rule in rules: - logging.info('%r', rule) - - if not rule.cidr: - # Eventually, a mechanism to grant access for security - # groups will turn up here. It'll use ipsets. - continue - - version = _get_ip_version(rule.cidr) - if version == 4: - rules = ipv4_rules - else: - rules = ipv6_rules - - protocol = rule.protocol - if version == 6 and rule.protocol == 'icmp': - protocol = 'icmpv6' - - args = ['-p', protocol, '-s', rule.cidr] - - if rule.protocol in ['udp', 'tcp']: - if rule.from_port == rule.to_port: - args += ['--dport', '%s' % (rule.from_port,)] - else: - args += ['-m', 'multiport', - '--dports', '%s:%s' % (rule.from_port, - rule.to_port)] - elif rule.protocol == 'icmp': - icmp_type = rule.from_port - icmp_code = rule.to_port - - if icmp_type == -1: - icmp_type_arg = None - else: - icmp_type_arg = '%s' % icmp_type - if not icmp_code == -1: - icmp_type_arg += '/%s' % icmp_code - - if icmp_type_arg: - if version == 4: - args += ['-m', 'icmp', '--icmp-type', - icmp_type_arg] - elif version == 6: - args += ['-m', 'icmp6', '--icmpv6-type', - icmp_type_arg] - - args += ['-j ACCEPT'] - rules += [' '.join(args)] - - ipv4_rules += ['-j $sg-fallback'] - ipv6_rules += ['-j $sg-fallback'] - - return ipv4_rules, ipv6_rules - - def instance_filter_exists(self, instance): - """Check nova-instance-instance-xxx exists""" - return self.nwfilter.instance_filter_exists(instance) - - def refresh_security_group_members(self, security_group): - pass - - def refresh_security_group_rules(self, security_group, network_info=None): - self.do_refresh_security_group_rules(security_group, network_info) - self.iptables.apply() - - @utils.synchronized('iptables', external=True) - def do_refresh_security_group_rules(self, - security_group, - network_info=None): - for instance in self.instances.values(): - self.remove_filters_for_instance(instance) - if not network_info: - network_info = _get_network_info(instance) - self.add_filters_for_instance(instance, network_info) - - def _security_group_chain_name(self, security_group_id): - return 'nova-sg-%s' % (security_group_id,) - - def _instance_chain_name(self, instance): - return 'inst-%s' % (instance['id'],) diff --git a/nova/virt/libvirt/firewall.py b/nova/virt/libvirt/firewall.py new file mode 100644 index 000000000..b99f2ffb0 --- /dev/null +++ b/nova/virt/libvirt/firewall.py @@ -0,0 +1,820 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from eventlet import tpool + +from nova import context +from nova import db +from nova import flags +from nova import log as logging +from nova import utils +from nova.virt.libvirt import netutils + + +LOG = logging.getLogger("nova.virt.libvirt.firewall") +FLAGS = flags.FLAGS + + +try: + import libvirt +except ImportError: + LOG.warn(_("Libvirt module could not be loaded. NWFilterFirewall will " + "not work correctly.")) + + +class FirewallDriver(object): + def prepare_instance_filter(self, instance, network_info=None): + """Prepare filters for the instance. + + At this point, the instance isn't running yet.""" + raise NotImplementedError() + + def unfilter_instance(self, instance): + """Stop filtering instance""" + raise NotImplementedError() + + def apply_instance_filter(self, instance): + """Apply instance filter. + + Once this method returns, the instance should be firewalled + appropriately. This method should as far as possible be a + no-op. It's vastly preferred to get everything set up in + prepare_instance_filter. + """ + raise NotImplementedError() + + def refresh_security_group_rules(self, + security_group_id, + network_info=None): + """Refresh security group rules from data store + + Gets called when a rule has been added to or removed from + the security group.""" + raise NotImplementedError() + + def refresh_security_group_members(self, security_group_id): + """Refresh security group members from data store + + Gets called when an instance gets added to or removed from + the security group.""" + raise NotImplementedError() + + def refresh_provider_fw_rules(self): + """Refresh common rules for all hosts/instances from data store. + + Gets called when a rule has been added to or removed from + the list of rules (via admin api). + + """ + raise NotImplementedError() + + def setup_basic_filtering(self, instance, network_info=None): + """Create rules to block spoofing and allow dhcp. + + This gets called when spawning an instance, before + :method:`prepare_instance_filter`. + + """ + raise NotImplementedError() + + def instance_filter_exists(self, instance): + """Check nova-instance-instance-xxx exists""" + raise NotImplementedError() + + +class NWFilterFirewall(FirewallDriver): + """ + This class implements a network filtering mechanism versatile + enough for EC2 style Security Group filtering by leveraging + libvirt's nwfilter. + + First, all instances get a filter ("nova-base-filter") applied. + This filter provides some basic security such as protection against + MAC spoofing, IP spoofing, and ARP spoofing. + + This filter drops all incoming ipv4 and ipv6 connections. + Outgoing connections are never blocked. + + Second, every security group maps to a nwfilter filter(*). + NWFilters can be updated at runtime and changes are applied + immediately, so changes to security groups can be applied at + runtime (as mandated by the spec). + + Security group rules are named "nova-secgroup-<id>" where <id> + is the internal id of the security group. They're applied only on + hosts that have instances in the security group in question. + + Updates to security groups are done by updating the data model + (in response to API calls) followed by a request sent to all + the nodes with instances in the security group to refresh the + security group. + + Each instance has its own NWFilter, which references the above + mentioned security group NWFilters. This was done because + interfaces can only reference one filter while filters can + reference multiple other filters. This has the added benefit of + actually being able to add and remove security groups from an + instance at run time. This functionality is not exposed anywhere, + though. + + Outstanding questions: + + The name is unique, so would there be any good reason to sync + the uuid across the nodes (by assigning it from the datamodel)? + + + (*) This sentence brought to you by the redundancy department of + redundancy. + + """ + + def __init__(self, get_connection, **kwargs): + self._libvirt_get_connection = get_connection + self.static_filters_configured = False + self.handle_security_groups = False + + def apply_instance_filter(self, instance): + """No-op. Everything is done in prepare_instance_filter""" + pass + + def _get_connection(self): + return self._libvirt_get_connection() + _conn = property(_get_connection) + + def nova_dhcp_filter(self): + """The standard allow-dhcp-server filter is an <ip> one, so it uses + ebtables to allow traffic through. Without a corresponding rule in + iptables, it'll get blocked anyway.""" + + return '''<filter name='nova-allow-dhcp-server' chain='ipv4'> + <uuid>891e4787-e5c0-d59b-cbd6-41bc3c6b36fc</uuid> + <rule action='accept' direction='out' + priority='100'> + <udp srcipaddr='0.0.0.0' + dstipaddr='255.255.255.255' + srcportstart='68' + dstportstart='67'/> + </rule> + <rule action='accept' direction='in' + priority='100'> + <udp srcipaddr='$DHCPSERVER' + srcportstart='67' + dstportstart='68'/> + </rule> + </filter>''' + + def nova_ra_filter(self): + return '''<filter name='nova-allow-ra-server' chain='root'> + <uuid>d707fa71-4fb5-4b27-9ab7-ba5ca19c8804</uuid> + <rule action='accept' direction='inout' + priority='100'> + <icmpv6 srcipaddr='$RASERVER'/> + </rule> + </filter>''' + + def setup_basic_filtering(self, instance, network_info=None): + """Set up basic filtering (MAC, IP, and ARP spoofing protection)""" + logging.info('called setup_basic_filtering in nwfilter') + + if not network_info: + network_info = netutils.get_network_info(instance) + + if self.handle_security_groups: + # No point in setting up a filter set that we'll be overriding + # anyway. + return + + logging.info('ensuring static filters') + self._ensure_static_filters() + + if instance['image_ref'] == str(FLAGS.vpn_image_id): + base_filter = 'nova-vpn' + else: + base_filter = 'nova-base' + + for (network, mapping) in network_info: + nic_id = mapping['mac'].replace(':', '') + instance_filter_name = self._instance_filter_name(instance, nic_id) + self._define_filter(self._filter_container(instance_filter_name, + [base_filter])) + + def _ensure_static_filters(self): + """Static filters are filters that have no need to be IP aware. + + There is no configuration or tuneability of these filters, so they + can be set up once and forgotten about. + + """ + + if self.static_filters_configured: + return + + self._define_filter(self._filter_container('nova-base', + ['no-mac-spoofing', + 'no-ip-spoofing', + 'no-arp-spoofing', + 'allow-dhcp-server'])) + self._define_filter(self._filter_container('nova-vpn', + ['allow-dhcp-server'])) + self._define_filter(self.nova_base_ipv4_filter) + self._define_filter(self.nova_base_ipv6_filter) + self._define_filter(self.nova_dhcp_filter) + self._define_filter(self.nova_ra_filter) + if FLAGS.allow_project_net_traffic: + self._define_filter(self.nova_project_filter) + if FLAGS.use_ipv6: + self._define_filter(self.nova_project_filter_v6) + + self.static_filters_configured = True + + def _filter_container(self, name, filters): + xml = '''<filter name='%s' chain='root'>%s</filter>''' % ( + name, + ''.join(["<filterref filter='%s'/>" % (f,) for f in filters])) + return xml + + def nova_base_ipv4_filter(self): + retval = "<filter name='nova-base-ipv4' chain='ipv4'>" + for protocol in ['tcp', 'udp', 'icmp']: + for direction, action, priority in [('out', 'accept', 399), + ('in', 'drop', 400)]: + retval += """<rule action='%s' direction='%s' priority='%d'> + <%s /> + </rule>""" % (action, direction, + priority, protocol) + retval += '</filter>' + return retval + + def nova_base_ipv6_filter(self): + retval = "<filter name='nova-base-ipv6' chain='ipv6'>" + for protocol in ['tcp-ipv6', 'udp-ipv6', 'icmpv6']: + for direction, action, priority in [('out', 'accept', 399), + ('in', 'drop', 400)]: + retval += """<rule action='%s' direction='%s' priority='%d'> + <%s /> + </rule>""" % (action, direction, + priority, protocol) + retval += '</filter>' + return retval + + def nova_project_filter(self): + retval = "<filter name='nova-project' chain='ipv4'>" + for protocol in ['tcp', 'udp', 'icmp']: + retval += """<rule action='accept' direction='in' priority='200'> + <%s srcipaddr='$PROJNET' srcipmask='$PROJMASK' /> + </rule>""" % protocol + retval += '</filter>' + return retval + + def nova_project_filter_v6(self): + retval = "<filter name='nova-project-v6' chain='ipv6'>" + for protocol in ['tcp-ipv6', 'udp-ipv6', 'icmpv6']: + retval += """<rule action='accept' direction='inout' + priority='200'> + <%s srcipaddr='$PROJNETV6' + srcipmask='$PROJMASKV6' /> + </rule>""" % (protocol) + retval += '</filter>' + return retval + + def _define_filter(self, xml): + if callable(xml): + xml = xml() + # execute in a native thread and block current greenthread until done + tpool.execute(self._conn.nwfilterDefineXML, xml) + + def unfilter_instance(self, instance): + """Clear out the nwfilter rules.""" + network_info = netutils.get_network_info(instance) + instance_name = instance.name + for (network, mapping) in network_info: + nic_id = mapping['mac'].replace(':', '') + instance_filter_name = self._instance_filter_name(instance, nic_id) + + try: + self._conn.nwfilterLookupByName(instance_filter_name).\ + undefine() + except libvirt.libvirtError: + LOG.debug(_('The nwfilter(%(instance_filter_name)s) ' + 'for %(instance_name)s is not found.') % locals()) + + instance_secgroup_filter_name =\ + '%s-secgroup' % (self._instance_filter_name(instance)) + + try: + self._conn.nwfilterLookupByName(instance_secgroup_filter_name)\ + .undefine() + except libvirt.libvirtError: + LOG.debug(_('The nwfilter(%(instance_secgroup_filter_name)s) ' + 'for %(instance_name)s is not found.') % locals()) + + def prepare_instance_filter(self, instance, network_info=None): + """Creates an NWFilter for the given instance. + + In the process, it makes sure the filters for the provider blocks, + security groups, and base filter are all in place. + + """ + if not network_info: + network_info = netutils.get_network_info(instance) + + self.refresh_provider_fw_rules() + + ctxt = context.get_admin_context() + + instance_secgroup_filter_name = \ + '%s-secgroup' % (self._instance_filter_name(instance)) + + instance_secgroup_filter_children = ['nova-base-ipv4', + 'nova-base-ipv6', + 'nova-allow-dhcp-server'] + + if FLAGS.use_ipv6: + networks = [network for (network, _m) in network_info if + network['gateway_v6']] + + if networks: + instance_secgroup_filter_children.\ + append('nova-allow-ra-server') + + for security_group in \ + db.security_group_get_by_instance(ctxt, instance['id']): + + self.refresh_security_group_rules(security_group['id']) + + instance_secgroup_filter_children.append('nova-secgroup-%s' % + security_group['id']) + + self._define_filter( + self._filter_container(instance_secgroup_filter_name, + instance_secgroup_filter_children)) + + network_filters = self.\ + _create_network_filters(instance, network_info, + instance_secgroup_filter_name) + + for (name, children) in network_filters: + self._define_filters(name, children) + + def _create_network_filters(self, instance, network_info, + instance_secgroup_filter_name): + if instance['image_ref'] == str(FLAGS.vpn_image_id): + base_filter = 'nova-vpn' + else: + base_filter = 'nova-base' + + result = [] + for (_n, mapping) in network_info: + nic_id = mapping['mac'].replace(':', '') + instance_filter_name = self._instance_filter_name(instance, nic_id) + instance_filter_children = [base_filter, 'nova-provider-rules', + instance_secgroup_filter_name] + + if FLAGS.allow_project_net_traffic: + instance_filter_children.append('nova-project') + if FLAGS.use_ipv6: + instance_filter_children.append('nova-project-v6') + + result.append((instance_filter_name, instance_filter_children)) + + return result + + def _define_filters(self, filter_name, filter_children): + self._define_filter(self._filter_container(filter_name, + filter_children)) + + def refresh_security_group_rules(self, + security_group_id, + network_info=None): + return self._define_filter( + self.security_group_to_nwfilter_xml(security_group_id)) + + def refresh_provider_fw_rules(self): + """Update rules for all instances. + + This is part of the FirewallDriver API and is called when the + provider firewall rules change in the database. In the + `prepare_instance_filter` we add a reference to the + 'nova-provider-rules' filter for each instance's firewall, and + by changing that filter we update them all. + + """ + xml = self.provider_fw_to_nwfilter_xml() + return self._define_filter(xml) + + def security_group_to_nwfilter_xml(self, security_group_id): + security_group = db.security_group_get(context.get_admin_context(), + security_group_id) + rule_xml = "" + v6protocol = {'tcp': 'tcp-ipv6', 'udp': 'udp-ipv6', 'icmp': 'icmpv6'} + for rule in security_group.rules: + rule_xml += "<rule action='accept' direction='in' priority='300'>" + if rule.cidr: + version = netutils.get_ip_version(rule.cidr) + if(FLAGS.use_ipv6 and version == 6): + net, prefixlen = netutils.get_net_and_prefixlen(rule.cidr) + rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \ + (v6protocol[rule.protocol], net, prefixlen) + else: + net, mask = netutils.get_net_and_mask(rule.cidr) + rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \ + (rule.protocol, net, mask) + if rule.protocol in ['tcp', 'udp']: + rule_xml += "dstportstart='%s' dstportend='%s' " % \ + (rule.from_port, rule.to_port) + elif rule.protocol == 'icmp': + LOG.info('rule.protocol: %r, rule.from_port: %r, ' + 'rule.to_port: %r', rule.protocol, + rule.from_port, rule.to_port) + if rule.from_port != -1: + rule_xml += "type='%s' " % rule.from_port + if rule.to_port != -1: + rule_xml += "code='%s' " % rule.to_port + + rule_xml += '/>\n' + rule_xml += "</rule>\n" + xml = "<filter name='nova-secgroup-%s' " % security_group_id + if(FLAGS.use_ipv6): + xml += "chain='root'>%s</filter>" % rule_xml + else: + xml += "chain='ipv4'>%s</filter>" % rule_xml + return xml + + def provider_fw_to_nwfilter_xml(self): + """Compose a filter of drop rules from specified cidrs.""" + rule_xml = "" + v6protocol = {'tcp': 'tcp-ipv6', 'udp': 'udp-ipv6', 'icmp': 'icmpv6'} + rules = db.provider_fw_rule_get_all(context.get_admin_context()) + for rule in rules: + rule_xml += "<rule action='block' direction='in' priority='150'>" + version = netutils.get_ip_version(rule.cidr) + if(FLAGS.use_ipv6 and version == 6): + net, prefixlen = netutils.get_net_and_prefixlen(rule.cidr) + rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \ + (v6protocol[rule.protocol], net, prefixlen) + else: + net, mask = netutils.get_net_and_mask(rule.cidr) + rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \ + (rule.protocol, net, mask) + if rule.protocol in ['tcp', 'udp']: + rule_xml += "dstportstart='%s' dstportend='%s' " % \ + (rule.from_port, rule.to_port) + elif rule.protocol == 'icmp': + LOG.info('rule.protocol: %r, rule.from_port: %r, ' + 'rule.to_port: %r', rule.protocol, + rule.from_port, rule.to_port) + if rule.from_port != -1: + rule_xml += "type='%s' " % rule.from_port + if rule.to_port != -1: + rule_xml += "code='%s' " % rule.to_port + + rule_xml += '/>\n' + rule_xml += "</rule>\n" + xml = "<filter name='nova-provider-rules' " + if(FLAGS.use_ipv6): + xml += "chain='root'>%s</filter>" % rule_xml + else: + xml += "chain='ipv4'>%s</filter>" % rule_xml + return xml + + def _instance_filter_name(self, instance, nic_id=None): + if not nic_id: + return 'nova-instance-%s' % (instance['name']) + return 'nova-instance-%s-%s' % (instance['name'], nic_id) + + def instance_filter_exists(self, instance): + """Check nova-instance-instance-xxx exists""" + network_info = netutils.get_network_info(instance) + for (network, mapping) in network_info: + nic_id = mapping['mac'].replace(':', '') + instance_filter_name = self._instance_filter_name(instance, nic_id) + try: + self._conn.nwfilterLookupByName(instance_filter_name) + except libvirt.libvirtError: + name = instance.name + LOG.debug(_('The nwfilter(%(instance_filter_name)s) for' + '%(name)s is not found.') % locals()) + return False + return True + + +class IptablesFirewallDriver(FirewallDriver): + def __init__(self, execute=None, **kwargs): + from nova.network import linux_net + self.iptables = linux_net.iptables_manager + self.instances = {} + self.nwfilter = NWFilterFirewall(kwargs['get_connection']) + self.basicly_filtered = False + + self.iptables.ipv4['filter'].add_chain('sg-fallback') + self.iptables.ipv4['filter'].add_rule('sg-fallback', '-j DROP') + self.iptables.ipv6['filter'].add_chain('sg-fallback') + self.iptables.ipv6['filter'].add_rule('sg-fallback', '-j DROP') + + def setup_basic_filtering(self, instance, network_info=None): + """Set up provider rules and basic NWFilter.""" + if not network_info: + network_info = netutils.get_network_info(instance) + self.nwfilter.setup_basic_filtering(instance, network_info) + if not self.basicly_filtered: + LOG.debug(_('iptables firewall: Setup Basic Filtering')) + self.refresh_provider_fw_rules() + self.basicly_filtered = True + + def apply_instance_filter(self, instance): + """No-op. Everything is done in prepare_instance_filter""" + pass + + def unfilter_instance(self, instance): + if self.instances.pop(instance['id'], None): + self.remove_filters_for_instance(instance) + self.iptables.apply() + self.nwfilter.unfilter_instance(instance) + else: + LOG.info(_('Attempted to unfilter instance %s which is not ' + 'filtered'), instance['id']) + + def prepare_instance_filter(self, instance, network_info=None): + if not network_info: + network_info = netutils.get_network_info(instance) + self.instances[instance['id']] = instance + self.add_filters_for_instance(instance, network_info) + self.iptables.apply() + + def _create_filter(self, ips, chain_name): + return ['-d %s -j $%s' % (ip, chain_name) for ip in ips] + + def _filters_for_instance(self, chain_name, network_info): + ips_v4 = [ip['ip'] for (_n, mapping) in network_info + for ip in mapping['ips']] + ipv4_rules = self._create_filter(ips_v4, chain_name) + + ipv6_rules = [] + if FLAGS.use_ipv6: + ips_v6 = [ip['ip'] for (_n, mapping) in network_info + for ip in mapping['ip6s']] + ipv6_rules = self._create_filter(ips_v6, chain_name) + + return ipv4_rules, ipv6_rules + + def _add_filters(self, chain_name, ipv4_rules, ipv6_rules): + for rule in ipv4_rules: + self.iptables.ipv4['filter'].add_rule(chain_name, rule) + + if FLAGS.use_ipv6: + for rule in ipv6_rules: + self.iptables.ipv6['filter'].add_rule(chain_name, rule) + + def add_filters_for_instance(self, instance, network_info=None): + chain_name = self._instance_chain_name(instance) + if FLAGS.use_ipv6: + self.iptables.ipv6['filter'].add_chain(chain_name) + self.iptables.ipv4['filter'].add_chain(chain_name) + ipv4_rules, ipv6_rules = self._filters_for_instance(chain_name, + network_info) + self._add_filters('local', ipv4_rules, ipv6_rules) + ipv4_rules, ipv6_rules = self.instance_rules(instance, network_info) + self._add_filters(chain_name, ipv4_rules, ipv6_rules) + + def remove_filters_for_instance(self, instance): + chain_name = self._instance_chain_name(instance) + + self.iptables.ipv4['filter'].remove_chain(chain_name) + if FLAGS.use_ipv6: + self.iptables.ipv6['filter'].remove_chain(chain_name) + + def instance_rules(self, instance, network_info=None): + if not network_info: + network_info = netutils.get_network_info(instance) + ctxt = context.get_admin_context() + + ipv4_rules = [] + ipv6_rules = [] + + # Always drop invalid packets + ipv4_rules += ['-m state --state ' 'INVALID -j DROP'] + ipv6_rules += ['-m state --state ' 'INVALID -j DROP'] + + # Allow established connections + ipv4_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT'] + ipv6_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT'] + + # Pass through provider-wide drops + ipv4_rules += ['-j $provider'] + ipv6_rules += ['-j $provider'] + + dhcp_servers = [network['gateway'] for (network, _m) in network_info] + + for dhcp_server in dhcp_servers: + ipv4_rules.append('-s %s -p udp --sport 67 --dport 68 ' + '-j ACCEPT' % (dhcp_server,)) + + #Allow project network traffic + if FLAGS.allow_project_net_traffic: + cidrs = [network['cidr'] for (network, _m) in network_info] + for cidr in cidrs: + ipv4_rules.append('-s %s -j ACCEPT' % (cidr,)) + + # We wrap these in FLAGS.use_ipv6 because they might cause + # a DB lookup. The other ones are just list operations, so + # they're not worth the clutter. + if FLAGS.use_ipv6: + # Allow RA responses + gateways_v6 = [network['gateway_v6'] for (network, _m) in + network_info] + for gateway_v6 in gateways_v6: + ipv6_rules.append( + '-s %s/128 -p icmpv6 -j ACCEPT' % (gateway_v6,)) + + #Allow project network traffic + if FLAGS.allow_project_net_traffic: + cidrv6s = [network['cidr_v6'] for (network, _m) + in network_info] + + for cidrv6 in cidrv6s: + ipv6_rules.append('-s %s -j ACCEPT' % (cidrv6,)) + + security_groups = db.security_group_get_by_instance(ctxt, + instance['id']) + + # then, security group chains and rules + for security_group in security_groups: + rules = db.security_group_rule_get_by_security_group(ctxt, + security_group['id']) + + for rule in rules: + LOG.debug(_('Adding security group rule: %r'), rule) + + if not rule.cidr: + # Eventually, a mechanism to grant access for security + # groups will turn up here. It'll use ipsets. + continue + + version = netutils.get_ip_version(rule.cidr) + if version == 4: + fw_rules = ipv4_rules + else: + fw_rules = ipv6_rules + + protocol = rule.protocol + if version == 6 and rule.protocol == 'icmp': + protocol = 'icmpv6' + + args = ['-p', protocol, '-s', rule.cidr] + + if rule.protocol in ['udp', 'tcp']: + if rule.from_port == rule.to_port: + args += ['--dport', '%s' % (rule.from_port,)] + else: + args += ['-m', 'multiport', + '--dports', '%s:%s' % (rule.from_port, + rule.to_port)] + elif rule.protocol == 'icmp': + icmp_type = rule.from_port + icmp_code = rule.to_port + + if icmp_type == -1: + icmp_type_arg = None + else: + icmp_type_arg = '%s' % icmp_type + if not icmp_code == -1: + icmp_type_arg += '/%s' % icmp_code + + if icmp_type_arg: + if version == 4: + args += ['-m', 'icmp', '--icmp-type', + icmp_type_arg] + elif version == 6: + args += ['-m', 'icmp6', '--icmpv6-type', + icmp_type_arg] + + args += ['-j ACCEPT'] + fw_rules += [' '.join(args)] + + ipv4_rules += ['-j $sg-fallback'] + ipv6_rules += ['-j $sg-fallback'] + + return ipv4_rules, ipv6_rules + + def instance_filter_exists(self, instance): + """Check nova-instance-instance-xxx exists""" + return self.nwfilter.instance_filter_exists(instance) + + def refresh_security_group_members(self, security_group): + pass + + def refresh_security_group_rules(self, security_group, network_info=None): + self.do_refresh_security_group_rules(security_group, network_info) + self.iptables.apply() + + @utils.synchronized('iptables', external=True) + def do_refresh_security_group_rules(self, + security_group, + network_info=None): + for instance in self.instances.values(): + self.remove_filters_for_instance(instance) + if not network_info: + network_info = netutils.get_network_info(instance) + self.add_filters_for_instance(instance, network_info) + + def refresh_provider_fw_rules(self): + """See class:FirewallDriver: docs.""" + self._do_refresh_provider_fw_rules() + self.iptables.apply() + + @utils.synchronized('iptables', external=True) + def _do_refresh_provider_fw_rules(self): + """Internal, synchronized version of refresh_provider_fw_rules.""" + self._purge_provider_fw_rules() + self._build_provider_fw_rules() + + def _purge_provider_fw_rules(self): + """Remove all rules from the provider chains.""" + self.iptables.ipv4['filter'].empty_chain('provider') + if FLAGS.use_ipv6: + self.iptables.ipv6['filter'].empty_chain('provider') + + def _build_provider_fw_rules(self): + """Create all rules for the provider IP DROPs.""" + self.iptables.ipv4['filter'].add_chain('provider') + if FLAGS.use_ipv6: + self.iptables.ipv6['filter'].add_chain('provider') + ipv4_rules, ipv6_rules = self._provider_rules() + for rule in ipv4_rules: + self.iptables.ipv4['filter'].add_rule('provider', rule) + + if FLAGS.use_ipv6: + for rule in ipv6_rules: + self.iptables.ipv6['filter'].add_rule('provider', rule) + + def _provider_rules(self): + """Generate a list of rules from provider for IP4 & IP6.""" + ctxt = context.get_admin_context() + ipv4_rules = [] + ipv6_rules = [] + rules = db.provider_fw_rule_get_all(ctxt) + for rule in rules: + LOG.debug(_('Adding provider rule: %s'), rule['cidr']) + version = netutils.get_ip_version(rule['cidr']) + if version == 4: + fw_rules = ipv4_rules + else: + fw_rules = ipv6_rules + + protocol = rule['protocol'] + if version == 6 and protocol == 'icmp': + protocol = 'icmpv6' + + args = ['-p', protocol, '-s', rule['cidr']] + + if protocol in ['udp', 'tcp']: + if rule['from_port'] == rule['to_port']: + args += ['--dport', '%s' % (rule['from_port'],)] + else: + args += ['-m', 'multiport', + '--dports', '%s:%s' % (rule['from_port'], + rule['to_port'])] + elif protocol == 'icmp': + icmp_type = rule['from_port'] + icmp_code = rule['to_port'] + + if icmp_type == -1: + icmp_type_arg = None + else: + icmp_type_arg = '%s' % icmp_type + if not icmp_code == -1: + icmp_type_arg += '/%s' % icmp_code + + if icmp_type_arg: + if version == 4: + args += ['-m', 'icmp', '--icmp-type', + icmp_type_arg] + elif version == 6: + args += ['-m', 'icmp6', '--icmpv6-type', + icmp_type_arg] + args += ['-j DROP'] + fw_rules += [' '.join(args)] + return ipv4_rules, ipv6_rules + + def _security_group_chain_name(self, security_group_id): + return 'nova-sg-%s' % (security_group_id,) + + def _instance_chain_name(self, instance): + return 'inst-%s' % (instance['id'],) diff --git a/nova/virt/libvirt/netutils.py b/nova/virt/libvirt/netutils.py new file mode 100644 index 000000000..0bad84f7c --- /dev/null +++ b/nova/virt/libvirt/netutils.py @@ -0,0 +1,97 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +"""Network-releated utilities for supporting libvirt connection code.""" + + +import netaddr + +from nova import context +from nova import db +from nova import flags +from nova import ipv6 +from nova import utils + + +FLAGS = flags.FLAGS + + +def get_net_and_mask(cidr): + net = netaddr.IPNetwork(cidr) + return str(net.ip), str(net.netmask) + + +def get_net_and_prefixlen(cidr): + net = netaddr.IPNetwork(cidr) + return str(net.ip), str(net._prefixlen) + + +def get_ip_version(cidr): + net = netaddr.IPNetwork(cidr) + return int(net.version) + + +def get_network_info(instance): + # TODO(adiantum) If we will keep this function + # we should cache network_info + admin_context = context.get_admin_context() + + ip_addresses = db.fixed_ip_get_all_by_instance(admin_context, + instance['id']) + networks = db.network_get_all_by_instance(admin_context, + instance['id']) + flavor = db.instance_type_get_by_id(admin_context, + instance['instance_type_id']) + network_info = [] + + for network in networks: + network_ips = [ip for ip in ip_addresses + if ip['network_id'] == network['id']] + + def ip_dict(ip): + return { + 'ip': ip['address'], + 'netmask': network['netmask'], + 'enabled': '1'} + + def ip6_dict(): + prefix = network['cidr_v6'] + mac = instance['mac_address'] + project_id = instance['project_id'] + return { + 'ip': ipv6.to_global(prefix, mac, project_id), + 'netmask': network['netmask_v6'], + 'enabled': '1'} + + mapping = { + 'label': network['label'], + 'gateway': network['gateway'], + 'broadcast': network['broadcast'], + 'mac': instance['mac_address'], + 'rxtx_cap': flavor['rxtx_cap'], + 'dns': [network['dns']], + 'ips': [ip_dict(ip) for ip in network_ips]} + + if FLAGS.use_ipv6: + mapping['ip6s'] = [ip6_dict()] + mapping['gateway6'] = network['gateway_v6'] + + network_info.append((network, mapping)) + return network_info diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py index c3e79a92f..5f76b0df5 100644 --- a/nova/virt/vmwareapi/vmops.py +++ b/nova/virt/vmwareapi/vmops.py @@ -150,7 +150,7 @@ class VMWareVMOps(object): """
image_size, image_properties = \
vmware_images.get_vmdk_size_and_properties(
- instance.image_id, instance)
+ instance.image_ref, instance)
vmdk_file_size_in_kb = int(image_size) / 1024
os_type = image_properties.get("vmware_ostype", "otherGuest")
adapter_type = image_properties.get("vmware_adaptertype",
@@ -265,23 +265,23 @@ class VMWareVMOps(object): def _fetch_image_on_esx_datastore():
"""Fetch image from Glance to ESX datastore."""
- LOG.debug(_("Downloading image file data %(image_id)s to the ESX "
+ LOG.debug(_("Downloading image file data %(image_ref)s to the ESX "
"data store %(data_store_name)s") %
- ({'image_id': instance.image_id,
+ ({'image_ref': instance.image_ref,
'data_store_name': data_store_name}))
# Upload the -flat.vmdk file whose meta-data file we just created
# above
vmware_images.fetch_image(
- instance.image_id,
+ instance.image_ref,
instance,
host=self._session._host_ip,
data_center_name=self._get_datacenter_name_and_ref()[1],
datastore_name=data_store_name,
cookies=cookies,
file_path=flat_uploaded_vmdk_name)
- LOG.debug(_("Downloaded image file data %(image_id)s to the ESX "
+ LOG.debug(_("Downloaded image file data %(image_ref)s to the ESX "
"data store %(data_store_name)s") %
- ({'image_id': instance.image_id,
+ ({'image_ref': instance.image_ref,
'data_store_name': data_store_name}))
_fetch_image_on_esx_datastore()
@@ -590,11 +590,11 @@ class VMWareVMOps(object): def pause(self, instance, callback):
"""Pause a VM instance."""
- raise exception.APIError("pause not supported for vmwareapi")
+ raise exception.ApiError("pause not supported for vmwareapi")
def unpause(self, instance, callback):
"""Un-Pause a VM instance."""
- raise exception.APIError("unpause not supported for vmwareapi")
+ raise exception.ApiError("unpause not supported for vmwareapi")
def suspend(self, instance, callback):
"""Suspend the specified instance."""
@@ -673,7 +673,7 @@ class VMWareVMOps(object): def get_diagnostics(self, instance):
"""Return data about VM diagnostics."""
- raise exception.APIError("get_diagnostics not implemented for "
+ raise exception.ApiError("get_diagnostics not implemented for "
"vmwareapi")
def get_console_output(self, instance):
diff --git a/nova/virt/vmwareapi/vmware_images.py b/nova/virt/vmwareapi/vmware_images.py index 50c6baedf..70adba74f 100644 --- a/nova/virt/vmwareapi/vmware_images.py +++ b/nova/virt/vmwareapi/vmware_images.py @@ -18,10 +18,9 @@ Utility functions for Image transfer.
"""
-from glance import client
-
from nova import exception
from nova import flags
+import nova.image
from nova import log as logging
from nova.virt.vmwareapi import io_util
from nova.virt.vmwareapi import read_write_util
@@ -91,8 +90,6 @@ def fetch_image(image, instance, **kwargs): func = _get_glance_image
elif FLAGS.image_service == "nova.image.s3.S3ImageService":
func = _get_s3_image
- elif FLAGS.image_service == "nova.image.local.LocalImageService":
- func = _get_local_image
else:
raise NotImplementedError(_("The Image Service %s is not implemented")
% FLAGS.image_service)
@@ -106,8 +103,6 @@ def upload_image(image, instance, **kwargs): func = _put_glance_image
elif FLAGS.image_service == "nova.image.s3.S3ImageService":
func = _put_s3_image
- elif FLAGS.image_service == "nova.image.local.LocalImageService":
- func = _put_local_image
else:
raise NotImplementedError(_("The Image Service %s is not implemented")
% FLAGS.image_service)
@@ -117,8 +112,8 @@ def upload_image(image, instance, **kwargs): def _get_glance_image(image, instance, **kwargs):
"""Download image from the glance image server."""
LOG.debug(_("Downloading image %s from glance image server") % image)
- glance_client = client.Client(FLAGS.glance_host, FLAGS.glance_port)
- metadata, read_iter = glance_client.get_image(image)
+ (glance_client, image_id) = nova.image.get_glance_client(image)
+ metadata, read_iter = glance_client.get_image(image_id)
read_file_handle = read_write_util.GlanceFileRead(read_iter)
file_size = int(metadata['size'])
write_file_handle = read_write_util.VMWareHTTPWriteFile(
@@ -153,7 +148,7 @@ def _put_glance_image(image, instance, **kwargs): kwargs.get("cookies"),
kwargs.get("file_path"))
file_size = read_file_handle.get_size()
- glance_client = client.Client(FLAGS.glance_host, FLAGS.glance_port)
+ (glance_client, image_id) = nova.image.get_glance_client(image)
# The properties and other fields that we need to set for the image.
image_metadata = {"is_public": True,
"disk_format": "vmdk",
@@ -165,7 +160,7 @@ def _put_glance_image(image, instance, **kwargs): "vmware_image_version":
kwargs.get("image_version")}}
start_transfer(read_file_handle, file_size, glance_client=glance_client,
- image_id=image, image_meta=image_metadata)
+ image_id=image_id, image_meta=image_metadata)
LOG.debug(_("Uploaded image %s to the Glance image server") % image)
@@ -188,14 +183,11 @@ def get_vmdk_size_and_properties(image, instance): LOG.debug(_("Getting image size for the image %s") % image)
if FLAGS.image_service == "nova.image.glance.GlanceImageService":
- glance_client = client.Client(FLAGS.glance_host,
- FLAGS.glance_port)
- meta_data = glance_client.get_image_meta(image)
+ (glance_client, image_id) = nova.image.get_glance_client(image)
+ meta_data = glance_client.get_image_meta(image_id)
size, properties = meta_data["size"], meta_data["properties"]
elif FLAGS.image_service == "nova.image.s3.S3ImageService":
raise NotImplementedError
- elif FLAGS.image_service == "nova.image.local.LocalImageService":
- raise NotImplementedError
LOG.debug(_("Got image size of %(size)s for the image %(image)s") %
locals())
return size, properties
diff --git a/nova/virt/vmwareapi_conn.py b/nova/virt/vmwareapi_conn.py index 1c6d2572d..3c6345ec8 100644 --- a/nova/virt/vmwareapi_conn.py +++ b/nova/virt/vmwareapi_conn.py @@ -124,7 +124,7 @@ class VMWareESXConnection(driver.ComputeDriver): """List VM instances."""
return self._vmops.list_instances()
- def spawn(self, instance):
+ def spawn(self, instance, network_info=None, block_device_mapping=None):
"""Create VM instance."""
self._vmops.spawn(instance)
diff --git a/nova/virt/xenapi/fake.py b/nova/virt/xenapi/fake.py index e36ef3288..d5ac39473 100644 --- a/nova/virt/xenapi/fake.py +++ b/nova/virt/xenapi/fake.py @@ -51,13 +51,13 @@ A fake XenAPI SDK. """ -import datetime import uuid from pprint import pformat from nova import exception from nova import log as logging +from nova import utils _CLASSES = ['host', 'network', 'session', 'SR', 'VBD', @@ -146,6 +146,7 @@ def create_vdi(name_label, read_only, sr_ref, sharable): def create_vbd(vm_ref, vdi_ref): vbd_rec = {'VM': vm_ref, 'VDI': vdi_ref, + 'userdevice': '0', 'currently_attached': False} vbd_ref = _create_object('VBD', vbd_rec) after_VBD_create(vbd_ref, vbd_rec) @@ -159,7 +160,10 @@ def after_VBD_create(vbd_ref, vbd_rec): vbd_rec['device'] = '' vm_ref = vbd_rec['VM'] vm_rec = _db_content['VM'][vm_ref] - vm_rec['VBDs'] = [vbd_ref] + if vm_rec.get('VBDs', None): + vm_rec['VBDs'].append(vbd_ref) + else: + vm_rec['VBDs'] = [vbd_ref] vm_name_label = _db_content['VM'][vm_ref]['name_label'] vbd_rec['vm_name_label'] = vm_name_label @@ -337,10 +341,6 @@ class SessionBase(object): return db_ref['xenstore_data'][key] = None - def network_get_all_records_where(self, _1, _2): - # TODO (salvatore-orlando): filter table on _2 - return _db_content['network'] - def VM_add_to_xenstore_data(self, _1, vm_ref, key, value): db_ref = _db_content['VM'][vm_ref] if not 'xenstore_data' in db_ref: @@ -351,7 +351,7 @@ class SessionBase(object): #Always return 12GB available return 12 * 1024 * 1024 * 1024 - def host_call_plugin(*args): + def host_call_plugin(self, *args): return 'herp' def network_get_all_records_where(self, _1, filter): @@ -537,7 +537,7 @@ class SessionBase(object): except Failure, exc: task['error_info'] = exc.details task['status'] = 'failed' - task['finished'] = datetime.datetime.now() + task['finished'] = utils.utcnow() return task_ref def _check_session(self, params): diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 31f26e0dc..0d08e459c 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -19,6 +19,7 @@ Helper methods for operations related to the management of VM records and their attributes like VDIs, VIFs, as well as their lookup functions. """ +import json import os import pickle import re @@ -31,6 +32,8 @@ from xml.dom import minidom import glance.client from nova import exception from nova import flags +import nova.image +from nova.image import glance as glance_image_service from nova import log as logging from nova import utils from nova.auth.manager import AuthManager @@ -156,7 +159,6 @@ class VMHelper(HelperBase): rec['PV_ramdisk'] = ramdisk else: # 2. Use kernel within the image - rec['PV_args'] = 'clocksource=jiffies' rec['PV_bootloader'] = 'pygrub' else: # 3. Using hardware virtualization @@ -307,19 +309,16 @@ class VMHelper(HelperBase): @classmethod def get_vdi_for_vm_safely(cls, session, vm_ref): - vdi_refs = VMHelper.lookup_vm_vdis(session, vm_ref) - if vdi_refs is None: - raise Exception(_("No VDIs found for VM %s") % vm_ref) - else: - num_vdis = len(vdi_refs) - if num_vdis != 1: - raise Exception( - _("Unexpected number of VDIs (%(num_vdis)s) found" - " for VM %(vm_ref)s") % locals()) - - vdi_ref = vdi_refs[0] - vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref) - return vdi_ref, vdi_rec + """Retrieves the primary VDI for a VM""" + vbd_refs = session.get_xenapi().VM.get_VBDs(vm_ref) + for vbd in vbd_refs: + vbd_rec = session.get_xenapi().VBD.get_record(vbd) + # Convention dictates the primary VDI will be userdevice 0 + if vbd_rec['userdevice'] == '0': + vdi_rec = session.get_xenapi().VDI.get_record(vbd_rec['VDI']) + return vbd_rec['VDI'], vdi_rec + raise exception.Error(_("No primary VDI found for" + "%(vm_ref)s") % locals()) @classmethod def create_snapshot(cls, session, instance_id, vm_ref, label): @@ -353,12 +352,6 @@ class VMHelper(HelperBase): return template_vm_ref, template_vdi_uuids @classmethod - def get_sr(cls, session, sr_label='slices'): - """Finds the SR named by the given name label and returns - the UUID""" - return session.call_xenapi('SR.get_by_name_label', sr_label)[0] - - @classmethod def get_sr_path(cls, session): """Return the path to our storage repository @@ -382,10 +375,12 @@ class VMHelper(HelperBase): os_type = instance.os_type or FLAGS.default_os_type + glance_host, glance_port = \ + glance_image_service.pick_glance_api_server() params = {'vdi_uuids': vdi_uuids, 'image_id': image_id, - 'glance_host': FLAGS.glance_host, - 'glance_port': FLAGS.glance_port, + 'glance_host': glance_host, + 'glance_port': glance_port, 'sr_path': cls.get_sr_path(session), 'os_type': os_type} @@ -419,6 +414,9 @@ class VMHelper(HelperBase): xenapi_image_service = ['glance', 'objectstore'] glance_address = 'address for glance services' glance_port = 'port for glance services' + + Returns: A single filename if image_type is KERNEL_RAMDISK + A list of dictionaries that describe VDIs, otherwise """ access = AuthManager().get_access_key(user, project) @@ -433,6 +431,10 @@ class VMHelper(HelperBase): @classmethod def _fetch_image_glance_vhd(cls, session, instance_id, image, access, image_type): + """Tell glance to download an image and put the VHDs into the SR + + Returns: A list of dictionaries that describe VDIs + """ LOG.debug(_("Asking xapi to fetch vhd image %(image)s") % locals()) @@ -443,26 +445,36 @@ class VMHelper(HelperBase): # here (under Python 2.6+) and pass them as arguments uuid_stack = [str(uuid.uuid4()) for i in xrange(2)] + glance_host, glance_port = \ + glance_image_service.pick_glance_api_server() params = {'image_id': image, - 'glance_host': FLAGS.glance_host, - 'glance_port': FLAGS.glance_port, + 'glance_host': glance_host, + 'glance_port': glance_port, 'uuid_stack': uuid_stack, 'sr_path': cls.get_sr_path(session)} kwargs = {'params': pickle.dumps(params)} task = session.async_call_plugin('glance', 'download_vhd', kwargs) - vdi_uuid = session.wait_for_task(task, instance_id) + result = session.wait_for_task(task, instance_id) + # 'download_vhd' will return a json encoded string containing + # a list of dictionaries describing VDIs. The dictionary will + # contain 'vdi_type' and 'vdi_uuid' keys. 'vdi_type' can be + # 'os' or 'swap' right now. + vdis = json.loads(result) + for vdi in vdis: + LOG.debug(_("xapi 'download_vhd' returned VDI of " + "type '%(vdi_type)s' with UUID '%(vdi_uuid)s'" % vdi)) cls.scan_sr(session, instance_id, sr_ref) + # Pull out the UUID of the first VDI + vdi_uuid = vdis[0]['vdi_uuid'] # Set the name-label to ease debugging vdi_ref = session.get_xenapi().VDI.get_by_uuid(vdi_uuid) - name_label = get_name_label_for_image(image) - session.get_xenapi().VDI.set_name_label(vdi_ref, name_label) + primary_name_label = get_name_label_for_image(image) + session.get_xenapi().VDI.set_name_label(vdi_ref, primary_name_label) - LOG.debug(_("xapi 'download_vhd' returned VDI UUID %(vdi_uuid)s") - % locals()) - return vdi_uuid + return vdis @classmethod def _fetch_image_glance_disk(cls, session, instance_id, image, access, @@ -474,6 +486,8 @@ class VMHelper(HelperBase): plugin; instead, it streams the disks through domU to the VDI directly. + Returns: A single filename if image_type is KERNEL_RAMDISK + A list of dictionaries that describe VDIs, otherwise """ # FIXME(sirp): Since the Glance plugin seems to be required for the # VHD disk, it may be worth using the plugin for both VHD and RAW and @@ -485,8 +499,8 @@ class VMHelper(HelperBase): else: sr_ref = safe_find_sr(session) - client = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) - meta, image_file = client.get_image(image) + glance_client, image_id = nova.image.get_glance_client(image) + meta, image_file = glance_client.get_image(image_id) virtual_size = int(meta['size']) vdi_size = virtual_size LOG.debug(_("Size for image %(image)s:%(virtual_size)d") % locals()) @@ -526,7 +540,8 @@ class VMHelper(HelperBase): LOG.debug(_("Kernel/Ramdisk VDI %s destroyed"), vdi_ref) return filename else: - return session.get_xenapi().VDI.get_uuid(vdi_ref) + vdi_uuid = session.get_xenapi().VDI.get_uuid(vdi_ref) + return [dict(vdi_type='os', vdi_uuid=vdi_uuid)] @classmethod def determine_disk_image_type(cls, instance): @@ -547,10 +562,10 @@ class VMHelper(HelperBase): ImageType.DISK_VHD: 'DISK_VHD', ImageType.DISK_ISO: 'DISK_ISO'} disk_format = pretty_format[image_type] - image_id = instance.image_id + image_ref = instance.image_ref instance_id = instance.id LOG.debug(_("Detected %(disk_format)s format for image " - "%(image_id)s, instance %(instance_id)s") % locals()) + "%(image_ref)s, instance %(instance_id)s") % locals()) def determine_from_glance(): glance_disk_format2nova_type = { @@ -560,8 +575,9 @@ class VMHelper(HelperBase): 'raw': ImageType.DISK_RAW, 'vhd': ImageType.DISK_VHD, 'iso': ImageType.DISK_ISO} - client = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) - meta = client.get_image_meta(instance.image_id) + image_ref = instance.image_ref + glance_client, image_id = nova.image.get_glance_client(image_ref) + meta = glance_client.get_image_meta(image_id) disk_format = meta['disk_format'] try: return glance_disk_format2nova_type[disk_format] @@ -587,6 +603,11 @@ class VMHelper(HelperBase): @classmethod def _fetch_image_glance(cls, session, instance_id, image, access, image_type): + """Fetch image from glance based on image type. + + Returns: A single filename if image_type is KERNEL_RAMDISK + A list of dictionaries that describe VDIs, otherwise + """ if image_type == ImageType.DISK_VHD: return cls._fetch_image_glance_vhd( session, instance_id, image, access, image_type) @@ -597,7 +618,13 @@ class VMHelper(HelperBase): @classmethod def _fetch_image_objectstore(cls, session, instance_id, image, access, secret, image_type): - url = images.image_url(image) + """Fetch an image from objectstore. + + Returns: A single filename if image_type is KERNEL_RAMDISK + A list of dictionaries that describe VDIs, otherwise + """ + url = "http://%s:%s/_images/%s/image" % (FLAGS.s3_host, FLAGS.s3_port, + image) LOG.debug(_("Asking xapi to fetch %(url)s as %(access)s") % locals()) if image_type == ImageType.KERNEL_RAMDISK: fn = 'get_kernel' @@ -614,8 +641,10 @@ class VMHelper(HelperBase): if image_type == ImageType.DISK_RAW: args['raw'] = 'true' task = session.async_call_plugin('objectstore', fn, args) - uuid = session.wait_for_task(task, instance_id) - return uuid + uuid_or_fn = session.wait_for_task(task, instance_id) + if image_type != ImageType.KERNEL_RAMDISK: + return [dict(vdi_type='os', vdi_uuid=uuid_or_fn)] + return uuid_or_fn @classmethod def determine_is_pv(cls, session, instance_id, vdi_ref, disk_image_type, @@ -810,8 +839,7 @@ class VMHelper(HelperBase): @classmethod def scan_default_sr(cls, session): """Looks for the system default SR and triggers a re-scan""" - #FIXME(sirp/mdietz): refactor scan_default_sr in there - sr_ref = cls.get_sr(session) + sr_ref = find_sr(session) session.call_xenapi('SR.scan', sr_ref) @@ -903,7 +931,8 @@ def get_vdi_for_vm_safely(session, vm_ref): else: num_vdis = len(vdi_refs) if num_vdis != 1: - raise Exception(_("Unexpected number of VDIs (%(num_vdis)s) found" + raise exception.Exception(_("Unexpected number of VDIs" + "(%(num_vdis)s) found" " for VM %(vm_ref)s") % locals()) vdi_ref = vdi_refs[0] @@ -1109,6 +1138,8 @@ def _stream_disk(dev, image_type, virtual_size, image_file): offset = MBR_SIZE_BYTES _write_partition(virtual_size, dev) + utils.execute('sudo', 'chown', os.getuid(), '/dev/%s' % dev) + with open('/dev/%s' % dev, 'wb') as f: f.seek(offset) for chunk in image_file: diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 94d3b4044..5cf99b9ac 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -47,6 +47,21 @@ LOG = logging.getLogger("nova.virt.xenapi.vmops") FLAGS = flags.FLAGS +def cmp_version(a, b): + """Compare two version strings (eg 0.0.1.10 > 0.0.1.9)""" + a = a.split('.') + b = b.split('.') + + # Compare each individual portion of both version strings + for va, vb in zip(a, b): + ret = int(va) - int(vb) + if ret: + return ret + + # Fallback to comparing length last + return len(a) - len(b) + + class VMOps(object): """ Management class for VM-related tasks @@ -91,7 +106,8 @@ class VMOps(object): def finish_resize(self, instance, disk_info): vdi_uuid = self.link_disks(instance, disk_info['base_copy'], disk_info['cow']) - vm_ref = self._create_vm(instance, vdi_uuid) + vm_ref = self._create_vm(instance, + [dict(vdi_type='os', vdi_uuid=vdi_uuid)]) self.resize_instance(instance, vdi_uuid) self._spawn(instance, vm_ref) @@ -100,29 +116,30 @@ class VMOps(object): if not vm_ref: vm_ref = VMHelper.lookup(self._session, instance.name) if vm_ref is None: - raise exception(_('Attempted to power on non-existent instance' + raise Exception(_('Attempted to power on non-existent instance' ' bad instance id %s') % instance.id) LOG.debug(_("Starting instance %s"), instance.name) self._session.call_xenapi('VM.start', vm_ref, False, False) - def _create_disk(self, instance): + def _create_disks(self, instance): user = AuthManager().get_user(instance.user_id) project = AuthManager().get_project(instance.project_id) disk_image_type = VMHelper.determine_disk_image_type(instance) - vdi_uuid = VMHelper.fetch_image(self._session, instance.id, - instance.image_id, user, project, disk_image_type) - return vdi_uuid + vdis = VMHelper.fetch_image(self._session, + instance.id, instance.image_ref, user, project, + disk_image_type) + return vdis def spawn(self, instance, network_info=None): - vdi_uuid = self._create_disk(instance) - vm_ref = self._create_vm(instance, vdi_uuid, network_info) + vdis = self._create_disks(instance) + vm_ref = self._create_vm(instance, vdis, network_info) self._spawn(instance, vm_ref) def spawn_rescue(self, instance): """Spawn a rescue instance.""" self.spawn(instance) - def _create_vm(self, instance, vdi_uuid, network_info=None): + def _create_vm(self, instance, vdis, network_info=None): """Create VM instance.""" instance_name = instance.name vm_ref = VMHelper.lookup(self._session, instance_name) @@ -141,43 +158,81 @@ class VMOps(object): user = AuthManager().get_user(instance.user_id) project = AuthManager().get_project(instance.project_id) - # Are we building from a pre-existing disk? - vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) - disk_image_type = VMHelper.determine_disk_image_type(instance) kernel = None if instance.kernel_id: kernel = VMHelper.fetch_image(self._session, instance.id, - instance.kernel_id, user, project, ImageType.KERNEL_RAMDISK) + instance.kernel_id, user, project, + ImageType.KERNEL_RAMDISK) ramdisk = None if instance.ramdisk_id: ramdisk = VMHelper.fetch_image(self._session, instance.id, - instance.ramdisk_id, user, project, ImageType.KERNEL_RAMDISK) + instance.ramdisk_id, user, project, + ImageType.KERNEL_RAMDISK) + # Create the VM ref and attach the first disk + first_vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', + vdis[0]['vdi_uuid']) + + vm_mode = instance.vm_mode and instance.vm_mode.lower() + if vm_mode == 'pv': + use_pv_kernel = True + elif vm_mode in ('hv', 'hvm'): + use_pv_kernel = False + vm_mode = 'hvm' # Normalize + else: + use_pv_kernel = VMHelper.determine_is_pv(self._session, + instance.id, first_vdi_ref, disk_image_type, + instance.os_type) + vm_mode = use_pv_kernel and 'pv' or 'hvm' + + if instance.vm_mode != vm_mode: + # Update database with normalized (or determined) value + db.instance_update(context.get_admin_context(), + instance['id'], {'vm_mode': vm_mode}) + + vm_ref = VMHelper.create_vm(self._session, instance, + kernel, ramdisk, use_pv_kernel) - use_pv_kernel = VMHelper.determine_is_pv(self._session, instance.id, - vdi_ref, disk_image_type, instance.os_type) - vm_ref = VMHelper.create_vm(self._session, instance, kernel, ramdisk, - use_pv_kernel) + # device 0 reserved for RW disk + userdevice = 0; # DISK_ISO needs two VBDs: the ISO disk and a blank RW disk if disk_image_type == ImageType.DISK_ISO: LOG.debug("detected ISO image type, going to create blank VM for " "install") - # device 0 reserved for RW disk, so use '1' - cd_vdi_ref = vdi_ref - VMHelper.create_cd_vbd(session=self._session, vm_ref=vm_ref, - vdi_ref=cd_vdi_ref, userdevice=1, bootable=True) - vdi_ref = VMHelper.fetch_blank_disk(session=self._session, + cd_vdi_ref = first_vdi_ref + first_vdi_ref = VMHelper.fetch_blank_disk(session=self._session, instance_type_id=instance.instance_type_id) VMHelper.create_vbd(session=self._session, vm_ref=vm_ref, - vdi_ref=vdi_ref, userdevice=0, bootable=False) + vdi_ref=first_vdi_ref, userdevice=userdevice, bootable=False) + + # device 1 reserved for rescue disk so use '2'. + userdevice = userdevice + 2 + VMHelper.create_cd_vbd(session=self._session, vm_ref=vm_ref, + vdi_ref=cd_vdi_ref, userdevice=userdevice, bootable=True) + + else: VMHelper.create_vbd(session=self._session, vm_ref=vm_ref, - vdi_ref=vdi_ref, userdevice=0, bootable=True) + vdi_ref=first_vdi_ref, userdevice=userdevice, bootable=True) + # userdevice 1 is reserved for rescue + userdevice = userdevice + 1 + + + # Attach any other disks + for vdi in vdis[1:]: + # vdi['vdi_type'] is either 'os' or 'swap', but we don't + # really care what it is right here. + vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', + vdi['vdi_uuid']) + VMHelper.create_vbd(session=self._session, vm_ref=vm_ref, + vdi_ref=vdi_ref, userdevice=userdevice, + bootable=False) + userdevice += 1 # TODO(tr3buchet) - check to make sure we have network info, otherwise # create it now. This goes away once nova-multi-nic hits. @@ -187,7 +242,7 @@ class VMOps(object): # Alter the image before VM start for, e.g. network injection if FLAGS.xenapi_inject_image: VMHelper.preconfigure_instance(self._session, instance, - vdi_ref, network_info) + first_vdi_ref, network_info) self.create_vifs(vm_ref, network_info) self.inject_network_info(instance, network_info, vm_ref) @@ -201,6 +256,34 @@ class VMOps(object): LOG.info(_('Spawning VM %(instance_name)s created %(vm_ref)s.') % locals()) + ctx = context.get_admin_context() + agent_build = db.agent_build_get_by_triple(ctx, 'xen', + instance.os_type, instance.architecture) + if agent_build: + LOG.info(_('Latest agent build for %(hypervisor)s/%(os)s' + \ + '/%(architecture)s is %(version)s') % agent_build) + else: + LOG.info(_('No agent build found for %(hypervisor)s/%(os)s' + \ + '/%(architecture)s') % { + 'hypervisor': 'xen', + 'os': instance.os_type, + 'architecture': instance.architecture}) + + def _check_agent_version(): + version = self.get_agent_version(instance) + if not version: + LOG.info(_('No agent version returned by instance')) + return + + LOG.info(_('Instance agent version: %s') % version) + if not agent_build: + return + + if cmp_version(version, agent_build['version']) < 0: + LOG.info(_('Updating Agent to %s') % agent_build['version']) + self.agent_update(instance, agent_build['url'], + agent_build['md5hash']) + def _inject_files(): injected_files = instance.injected_files if injected_files: @@ -217,6 +300,13 @@ class VMOps(object): for path, contents in instance.injected_files: LOG.debug(_("Injecting file path: '%s'") % path) self.inject_file(instance, path, contents) + + def _set_admin_password(): + admin_password = instance.admin_pass + if admin_password: + LOG.debug(_("Setting admin password")) + self.set_admin_password(instance, admin_password) + # NOTE(armando): Do we really need to do this in virt? # NOTE(tr3buchet): not sure but wherever we do it, we need to call # reset_network afterwards @@ -228,7 +318,9 @@ class VMOps(object): if state == power_state.RUNNING: LOG.debug(_('Instance %s: booted'), instance_name) timer.stop() + _check_agent_version() _inject_files() + _set_admin_password() return True except Exception, exc: LOG.warn(exc) @@ -268,7 +360,8 @@ class VMOps(object): instance_name = instance_or_vm.name vm_ref = VMHelper.lookup(self._session, instance_name) if vm_ref is None: - raise exception.InstanceNotFound(instance_id=instance_obj.id) + raise exception.NotFound(_("No opaque_ref could be determined " + "for '%s'.") % instance_or_vm) return vm_ref def _acquire_bootlock(self, vm): @@ -432,6 +525,34 @@ class VMOps(object): task = self._session.call_xenapi('Async.VM.clean_reboot', vm_ref) self._session.wait_for_task(task, instance.id) + def get_agent_version(self, instance): + """Get the version of the agent running on the VM instance.""" + + # Send the encrypted password + transaction_id = str(uuid.uuid4()) + args = {'id': transaction_id} + resp = self._make_agent_call('version', instance, '', args) + if resp is None: + # No response from the agent + return + resp_dict = json.loads(resp) + return resp_dict['message'] + + def agent_update(self, instance, url, md5sum): + """Update agent on the VM instance.""" + + # Send the encrypted password + transaction_id = str(uuid.uuid4()) + args = {'id': transaction_id, 'url': url, 'md5sum': md5sum} + resp = self._make_agent_call('agentupdate', instance, '', args) + if resp is None: + # No response from the agent + return + resp_dict = json.loads(resp) + if resp_dict['returncode'] != '0': + raise RuntimeError(resp_dict['message']) + return resp_dict['message'] + def set_admin_password(self, instance, new_pass): """Set the root/admin password on the VM instance. @@ -472,6 +593,9 @@ class VMOps(object): # Successful return code from password is '0' if resp_dict['returncode'] != '0': raise RuntimeError(resp_dict['message']) + db.instance_update(context.get_admin_context(), + instance['id'], + dict(admin_pass=new_pass)) return resp_dict['message'] def inject_file(self, instance, path, contents): @@ -1176,16 +1300,12 @@ class SimpleDH(object): mpi = M2Crypto.m2.bn_to_mpi(bn) return mpi - def _run_ssl(self, text, which): - base_cmd = ('openssl enc -aes-128-cbc -a -pass pass:%(shared)s ' - '-nosalt %(dec_flag)s') - if which.lower()[0] == 'd': - dec_flag = ' -d' - else: - dec_flag = '' - shared = self._shared - cmd = base_cmd % locals() - proc = _runproc(cmd) + def _run_ssl(self, text, extra_args=None): + if not extra_args: + extra_args = '' + cmd = 'enc -aes-128-cbc -A -a -pass pass:%s -nosalt %s' % ( + self._shared, extra_args) + proc = _runproc('openssl %s' % cmd) proc.stdin.write(text) proc.stdin.close() proc.wait() @@ -1195,7 +1315,7 @@ class SimpleDH(object): return proc.stdout.read() def encrypt(self, text): - return self._run_ssl(text, 'enc') + return self._run_ssl(text).strip('\n') def decrypt(self, text): - return self._run_ssl(text, 'dec') + return self._run_ssl(text, '-d') diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 6d828e109..5fcec1715 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -194,7 +194,7 @@ class XenAPIConnection(driver.ComputeDriver): def list_instances_detail(self): return self._vmops.list_instances_detail() - def spawn(self, instance): + def spawn(self, instance, network_info=None, block_device_mapping=None): """Create VM instance""" self._vmops.spawn(instance) diff --git a/nova/vnc/__init__.py b/nova/vnc/__init__.py index b5b00e44e..859bfd65f 100644 --- a/nova/vnc/__init__.py +++ b/nova/vnc/__init__.py @@ -32,3 +32,5 @@ flags.DEFINE_string('vncserver_host', '0.0.0.0', 'the host interface on which vnc server should listen') flags.DEFINE_bool('vnc_enabled', True, 'enable vnc related features') +flags.DEFINE_string('vnc_keymap', 'en-us', + 'keymap for vnc') diff --git a/nova/volume/api.py b/nova/volume/api.py index 09befb647..7d27abff9 100644 --- a/nova/volume/api.py +++ b/nova/volume/api.py @@ -20,7 +20,8 @@ Handles all requests relating to volumes. """ -import datetime + +from eventlet import greenthread from nova import db from nova import exception @@ -28,6 +29,7 @@ from nova import flags from nova import log as logging from nova import quota from nova import rpc +from nova import utils from nova.db import base FLAGS = flags.FLAGS @@ -39,7 +41,15 @@ LOG = logging.getLogger('nova.volume') class API(base.Base): """API for interacting with the volume manager.""" - def create(self, context, size, name, description): + def create(self, context, size, snapshot_id, name, description): + if snapshot_id != None: + snapshot = self.get_snapshot(context, snapshot_id) + if snapshot['status'] != "available": + raise exception.ApiError( + _("Snapshot status must be available")) + if not size: + size = snapshot['volume_size'] + if quota.allowed_volumes(context, 1, size) < 1: pid = context.project_id LOG.warn(_("Quota exceeeded for %(pid)s, tried to create" @@ -51,6 +61,7 @@ class API(base.Base): 'size': size, 'user_id': context.user_id, 'project_id': context.project_id, + 'snapshot_id': snapshot_id, 'availability_zone': FLAGS.storage_availability_zone, 'status': "creating", 'attach_status': "detached", @@ -62,14 +73,23 @@ class API(base.Base): FLAGS.scheduler_topic, {"method": "create_volume", "args": {"topic": FLAGS.volume_topic, - "volume_id": volume['id']}}) + "volume_id": volume['id'], + "snapshot_id": snapshot_id}}) return volume + # TODO(yamahata): eliminate dumb polling + def wait_creation(self, context, volume_id): + while True: + volume = self.get(context, volume_id) + if volume['status'] != 'creating': + return + greenthread.sleep(1) + def delete(self, context, volume_id): volume = self.get(context, volume_id) if volume['status'] != "available": raise exception.ApiError(_("Volume status must be available")) - now = datetime.datetime.utcnow() + now = utils.utcnow() self.db.volume_update(context, volume_id, {'status': 'deleting', 'terminated_at': now}) host = volume['host'] @@ -90,6 +110,15 @@ class API(base.Base): return self.db.volume_get_all(context) return self.db.volume_get_all_by_project(context, context.project_id) + def get_snapshot(self, context, snapshot_id): + rv = self.db.snapshot_get(context, snapshot_id) + return dict(rv.iteritems()) + + def get_all_snapshots(self, context): + if context.is_admin: + return self.db.snapshot_get_all(context) + return self.db.snapshot_get_all_by_project(context, context.project_id) + def check_attach(self, context, volume_id): volume = self.get(context, volume_id) # TODO(vish): abstract status checking? @@ -110,3 +139,38 @@ class API(base.Base): self.db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "remove_volume", "args": {'volume_id': volume_id}}) + + def create_snapshot(self, context, volume_id, name, description): + volume = self.get(context, volume_id) + if volume['status'] != "available": + raise exception.ApiError(_("Volume status must be available")) + + options = { + 'volume_id': volume_id, + 'user_id': context.user_id, + 'project_id': context.project_id, + 'status': "creating", + 'progress': '0%', + 'volume_size': volume['size'], + 'display_name': name, + 'display_description': description} + + snapshot = self.db.snapshot_create(context, options) + rpc.cast(context, + FLAGS.scheduler_topic, + {"method": "create_snapshot", + "args": {"topic": FLAGS.volume_topic, + "volume_id": volume_id, + "snapshot_id": snapshot['id']}}) + return snapshot + + def delete_snapshot(self, context, snapshot_id): + snapshot = self.get_snapshot(context, snapshot_id) + if snapshot['status'] != "available": + raise exception.ApiError(_("Snapshot status must be available")) + self.db.snapshot_update(context, snapshot_id, {'status': 'deleting'}) + rpc.cast(context, + FLAGS.scheduler_topic, + {"method": "delete_snapshot", + "args": {"topic": FLAGS.volume_topic, + "snapshot_id": snapshot_id}}) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 55307ad9b..23e845deb 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -90,42 +90,97 @@ class VolumeDriver(object): raise exception.Error(_("volume group %s doesn't exist") % FLAGS.volume_group) - def create_volume(self, volume): - """Creates a logical volume. Can optionally return a Dictionary of - changes to the volume object to be persisted.""" - if int(volume['size']) == 0: - sizestr = '100M' - else: - sizestr = '%sG' % volume['size'] + def _create_volume(self, volume_name, sizestr): self._try_execute('sudo', 'lvcreate', '-L', sizestr, '-n', - volume['name'], - FLAGS.volume_group) + volume_name, FLAGS.volume_group) - def delete_volume(self, volume): - """Deletes a logical volume.""" + def _copy_volume(self, srcstr, deststr, size_in_g): + self._execute('sudo', 'dd', 'if=%s' % srcstr, 'of=%s' % deststr, + 'count=%d' % (size_in_g * 1024), 'bs=1M') + + def _volume_not_present(self, volume_name): + path_name = '%s/%s' % (FLAGS.volume_group, volume_name) try: - self._try_execute('sudo', 'lvdisplay', - '%s/%s' % - (FLAGS.volume_group, - volume['name'])) + self._try_execute('sudo', 'lvdisplay', path_name) except Exception as e: - # If the volume isn't present, then don't attempt to delete + # If the volume isn't present return True + return False + def _delete_volume(self, volume, size_in_g): + """Deletes a logical volume.""" # zero out old volumes to prevent data leaking between users # TODO(ja): reclaiming space should be done lazy and low priority - self._execute('sudo', 'dd', 'if=/dev/zero', - 'of=%s' % self.local_path(volume), - 'count=%d' % (volume['size'] * 1024), - 'bs=1M') + self._copy_volume('/dev/zero', self.local_path(volume), size_in_g) self._try_execute('sudo', 'lvremove', '-f', "%s/%s" % (FLAGS.volume_group, - volume['name'])) + self._escape_snapshot(volume['name']))) + + def _sizestr(self, size_in_g): + if int(size_in_g) == 0: + return '100M' + return '%sG' % size_in_g + + # Linux LVM reserves name that starts with snapshot, so that + # such volume name can't be created. Mangle it. + def _escape_snapshot(self, snapshot_name): + if not snapshot_name.startswith('snapshot'): + return snapshot_name + return '_' + snapshot_name + + def create_volume(self, volume): + """Creates a logical volume. Can optionally return a Dictionary of + changes to the volume object to be persisted.""" + self._create_volume(volume['name'], self._sizestr(volume['size'])) + + def create_volume_from_snapshot(self, volume, snapshot): + """Creates a volume from a snapshot.""" + self._create_volume(volume['name'], self._sizestr(volume['size'])) + self._copy_volume(self.local_path(snapshot), self.local_path(volume), + snapshot['volume_size']) + + def delete_volume(self, volume): + """Deletes a logical volume.""" + if self._volume_not_present(volume['name']): + # If the volume isn't present, then don't attempt to delete + return True + + # TODO(yamahata): lvm can't delete origin volume only without + # deleting derived snapshots. Can we do something fancy? + out, err = self._execute('sudo', 'lvdisplay', '--noheading', + '-C', '-o', 'Attr', + '%s/%s' % (FLAGS.volume_group, + volume['name'])) + # fake_execute returns None resulting unit test error + if out: + out = out.strip() + if (out[0] == 'o') or (out[0] == 'O'): + raise exception.VolumeIsBusy(volume_name=volume['name']) + + self._delete_volume(volume, volume['size']) + + def create_snapshot(self, snapshot): + """Creates a snapshot.""" + orig_lv_name = "%s/%s" % (FLAGS.volume_group, snapshot['volume_name']) + self._try_execute('sudo', 'lvcreate', '-L', + self._sizestr(snapshot['volume_size']), + '--name', self._escape_snapshot(snapshot['name']), + '--snapshot', orig_lv_name) + + def delete_snapshot(self, snapshot): + """Deletes a snapshot.""" + if self._volume_not_present(self._escape_snapshot(snapshot['name'])): + # If the snapshot isn't present, then don't attempt to delete + return True + + # TODO(yamahata): zeroing out the whole snapshot triggers COW. + # it's quite slow. + self._delete_volume(snapshot, snapshot['volume_size']) def local_path(self, volume): # NOTE(vish): stops deprecation warning escaped_group = FLAGS.volume_group.replace('-', '--') - escaped_name = volume['name'].replace('-', '--') + escaped_name = self._escape_snapshot(volume['name']).replace('-', '--') return "/dev/mapper/%s-%s" % (escaped_group, escaped_name) def ensure_export(self, context, volume): @@ -527,6 +582,14 @@ class FakeISCSIDriver(ISCSIDriver): """No setup necessary in fake mode.""" pass + def discover_volume(self, context, volume): + """Discover volume on a remote host.""" + return "/dev/disk/by-path/volume-id-%d" % volume['id'] + + def undiscover_volume(self, volume): + """Undiscover volume on a remote host.""" + pass + @staticmethod def fake_execute(cmd, *_args, **_kwargs): """Execute that simply logs the command.""" @@ -559,6 +622,18 @@ class RBDDriver(VolumeDriver): self._try_execute('rbd', '--pool', FLAGS.rbd_pool, 'rm', volume['name']) + def create_snapshot(self, snapshot): + """Creates an rbd snapshot""" + self._try_execute('rbd', '--pool', FLAGS.rbd_pool, + 'snap', 'create', '--snap', snapshot['name'], + snapshot['volume_name']) + + def delete_snapshot(self, snapshot): + """Deletes an rbd snapshot""" + self._try_execute('rbd', '--pool', FLAGS.rbd_pool, + 'snap', 'rm', '--snap', snapshot['name'], + snapshot['volume_name']) + def local_path(self, volume): """Returns the path of the rbd volume.""" # This is the same as the remote path @@ -600,18 +675,31 @@ class SheepdogDriver(VolumeDriver): def create_volume(self, volume): """Creates a sheepdog volume""" - if int(volume['size']) == 0: - sizestr = '100M' - else: - sizestr = '%sG' % volume['size'] self._try_execute('qemu-img', 'create', "sheepdog:%s" % volume['name'], - sizestr) + self._sizestr(volume['size'])) + + def create_volume_from_snapshot(self, volume, snapshot): + """Creates a sheepdog volume from a snapshot.""" + self._try_execute('qemu-img', 'create', '-b', + "sheepdog:%s:%s" % (snapshot['volume_name'], + snapshot['name']), + "sheepdog:%s" % volume['name']) def delete_volume(self, volume): """Deletes a logical volume""" self._try_execute('collie', 'vdi', 'delete', volume['name']) + def create_snapshot(self, snapshot): + """Creates a sheepdog snapshot""" + self._try_execute('qemu-img', 'snapshot', '-c', snapshot['name'], + "sheepdog:%s" % snapshot['volume_name']) + + def delete_snapshot(self, snapshot): + """Deletes a sheepdog snapshot""" + self._try_execute('collie', 'vdi', 'delete', snapshot['volume_name'], + '-s', snapshot['name']) + def local_path(self, volume): return "sheepdog:%s" % volume['name'] diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 2178389ce..798bd379a 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -42,8 +42,6 @@ intact. """ -import datetime - from nova import context from nova import exception @@ -90,7 +88,7 @@ class VolumeManager(manager.SchedulerDependentManager): else: LOG.info(_("volume %s: skipping export"), volume['name']) - def create_volume(self, context, volume_id): + def create_volume(self, context, volume_id, snapshot_id=None): """Creates and exports the volume.""" context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) @@ -108,7 +106,13 @@ class VolumeManager(manager.SchedulerDependentManager): vol_size = volume_ref['size'] LOG.debug(_("volume %(vol_name)s: creating lv of" " size %(vol_size)sG") % locals()) - model_update = self.driver.create_volume(volume_ref) + if snapshot_id == None: + model_update = self.driver.create_volume(volume_ref) + else: + snapshot_ref = self.db.snapshot_get(context, snapshot_id) + model_update = self.driver.create_volume_from_snapshot( + volume_ref, + snapshot_ref) if model_update: self.db.volume_update(context, volume_ref['id'], model_update) @@ -121,7 +125,7 @@ class VolumeManager(manager.SchedulerDependentManager): volume_ref['id'], {'status': 'error'}) raise - now = datetime.datetime.utcnow() + now = utils.utcnow() self.db.volume_update(context, volume_ref['id'], {'status': 'available', 'launched_at': now}) @@ -142,6 +146,12 @@ class VolumeManager(manager.SchedulerDependentManager): self.driver.remove_export(context, volume_ref) LOG.debug(_("volume %s: deleting"), volume_ref['name']) self.driver.delete_volume(volume_ref) + except exception.VolumeIsBusy, e: + LOG.debug(_("volume %s: volume is busy"), volume_ref['name']) + self.driver.ensure_export(context, volume_ref) + self.db.volume_update(context, volume_ref['id'], + {'status': 'available'}) + return True except Exception: self.db.volume_update(context, volume_ref['id'], @@ -152,6 +162,49 @@ class VolumeManager(manager.SchedulerDependentManager): LOG.debug(_("volume %s: deleted successfully"), volume_ref['name']) return True + def create_snapshot(self, context, volume_id, snapshot_id): + """Creates and exports the snapshot.""" + context = context.elevated() + snapshot_ref = self.db.snapshot_get(context, snapshot_id) + LOG.info(_("snapshot %s: creating"), snapshot_ref['name']) + + try: + snap_name = snapshot_ref['name'] + LOG.debug(_("snapshot %(snap_name)s: creating") % locals()) + model_update = self.driver.create_snapshot(snapshot_ref) + if model_update: + self.db.snapshot_update(context, snapshot_ref['id'], + model_update) + + except Exception: + self.db.snapshot_update(context, + snapshot_ref['id'], {'status': 'error'}) + raise + + self.db.snapshot_update(context, + snapshot_ref['id'], {'status': 'available', + 'progress': '100%'}) + LOG.debug(_("snapshot %s: created successfully"), snapshot_ref['name']) + return snapshot_id + + def delete_snapshot(self, context, snapshot_id): + """Deletes and unexports snapshot.""" + context = context.elevated() + snapshot_ref = self.db.snapshot_get(context, snapshot_id) + + try: + LOG.debug(_("snapshot %s: deleting"), snapshot_ref['name']) + self.driver.delete_snapshot(snapshot_ref) + except Exception: + self.db.snapshot_update(context, + snapshot_ref['id'], + {'status': 'error_deleting'}) + raise + + self.db.snapshot_destroy(context, snapshot_id) + LOG.debug(_("snapshot %s: deleted successfully"), snapshot_ref['name']) + return True + def setup_compute_volume(self, context, volume_id): """Setup remote volume on compute host. diff --git a/nova/wsgi.py b/nova/wsgi.py index ea9bb963d..33ba852bc 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -85,36 +85,7 @@ class Server(object): class Request(webob.Request): - - def best_match_content_type(self): - """Determine the most acceptable content-type. - - Based on the query extension then the Accept header. - - """ - parts = self.path.rsplit('.', 1) - - if len(parts) > 1: - format = parts[1] - if format in ['json', 'xml']: - return 'application/{0}'.format(parts[1]) - - ctypes = ['application/json', 'application/xml'] - bm = self.accept.best_match(ctypes) - - return bm or 'application/json' - - def get_content_type(self): - allowed_types = ("application/xml", "application/json") - if not "Content-Type" in self.headers: - msg = _("Missing Content-Type") - LOG.debug(msg) - raise webob.exc.HTTPBadRequest(msg) - type = self.content_type - if type in allowed_types: - return type - LOG.debug(_("Wrong Content-Type: %s") % type) - raise webob.exc.HTTPBadRequest("Invalid content type") + pass class Application(object): @@ -289,8 +260,8 @@ class Router(object): Each route in `mapper` must specify a 'controller', which is a WSGI app to call. You'll probably want to specify an 'action' as - well and have your controller be a wsgi.Controller, who will route - the request to the action method. + well and have your controller be an object that can route + the request to the action-specific method. Examples: mapper = routes.Mapper() @@ -338,223 +309,6 @@ class Router(object): return app -class Controller(object): - """WSGI app that dispatched to methods. - - WSGI app that reads routing information supplied by RoutesMiddleware - and calls the requested action method upon itself. All action methods - must, in addition to their normal parameters, accept a 'req' argument - which is the incoming wsgi.Request. They raise a webob.exc exception, - or return a dict which will be serialized by requested content type. - - """ - - @webob.dec.wsgify(RequestClass=Request) - def __call__(self, req): - """Call the method specified in req.environ by RoutesMiddleware.""" - arg_dict = req.environ['wsgiorg.routing_args'][1] - action = arg_dict['action'] - method = getattr(self, action) - LOG.debug("%s %s" % (req.method, req.url)) - del arg_dict['controller'] - del arg_dict['action'] - if 'format' in arg_dict: - del arg_dict['format'] - arg_dict['req'] = req - result = method(**arg_dict) - - if type(result) is dict: - content_type = req.best_match_content_type() - default_xmlns = self.get_default_xmlns(req) - body = self._serialize(result, content_type, default_xmlns) - - response = webob.Response() - response.headers['Content-Type'] = content_type - response.body = body - msg_dict = dict(url=req.url, status=response.status_int) - msg = _("%(url)s returned with HTTP %(status)d") % msg_dict - LOG.debug(msg) - return response - else: - return result - - def _serialize(self, data, content_type, default_xmlns): - """Serialize the given dict to the provided content_type. - - Uses self._serialization_metadata if it exists, which is a dict mapping - MIME types to information needed to serialize to that type. - - """ - _metadata = getattr(type(self), '_serialization_metadata', {}) - - serializer = Serializer(_metadata, default_xmlns) - try: - return serializer.serialize(data, content_type) - except exception.InvalidContentType: - raise webob.exc.HTTPNotAcceptable() - - def _deserialize(self, data, content_type): - """Deserialize the request body to the specefied content type. - - Uses self._serialization_metadata if it exists, which is a dict mapping - MIME types to information needed to serialize to that type. - - """ - _metadata = getattr(type(self), '_serialization_metadata', {}) - serializer = Serializer(_metadata) - return serializer.deserialize(data, content_type) - - def get_default_xmlns(self, req): - """Provide the XML namespace to use if none is otherwise specified.""" - return None - - -class Serializer(object): - """Serializes and deserializes dictionaries to certain MIME types.""" - - def __init__(self, metadata=None, default_xmlns=None): - """Create a serializer based on the given WSGI environment. - - 'metadata' is an optional dict mapping MIME types to information - needed to serialize a dictionary to that type. - - """ - self.metadata = metadata or {} - self.default_xmlns = default_xmlns - - def _get_serialize_handler(self, content_type): - handlers = { - 'application/json': self._to_json, - 'application/xml': self._to_xml, - } - - try: - return handlers[content_type] - except Exception: - raise exception.InvalidContentType(content_type=content_type) - - def serialize(self, data, content_type): - """Serialize a dictionary into the specified content type.""" - return self._get_serialize_handler(content_type)(data) - - def deserialize(self, datastring, content_type): - """Deserialize a string to a dictionary. - - The string must be in the format of a supported MIME type. - - """ - return self.get_deserialize_handler(content_type)(datastring) - - def get_deserialize_handler(self, content_type): - handlers = { - 'application/json': self._from_json, - 'application/xml': self._from_xml, - } - - try: - return handlers[content_type] - except Exception: - raise exception.InvalidContentType(content_type=content_type) - - def _from_json(self, datastring): - return utils.loads(datastring) - - def _from_xml(self, datastring): - xmldata = self.metadata.get('application/xml', {}) - plurals = set(xmldata.get('plurals', {})) - node = minidom.parseString(datastring).childNodes[0] - return {node.nodeName: self._from_xml_node(node, plurals)} - - def _from_xml_node(self, node, listnames): - """Convert a minidom node to a simple Python type. - - listnames is a collection of names of XML nodes whose subnodes should - be considered list items. - - """ - if len(node.childNodes) == 1 and node.childNodes[0].nodeType == 3: - return node.childNodes[0].nodeValue - elif node.nodeName in listnames: - return [self._from_xml_node(n, listnames) for n in node.childNodes] - else: - result = dict() - for attr in node.attributes.keys(): - result[attr] = node.attributes[attr].nodeValue - for child in node.childNodes: - if child.nodeType != node.TEXT_NODE: - result[child.nodeName] = self._from_xml_node(child, - listnames) - return result - - def _to_json(self, data): - return utils.dumps(data) - - def _to_xml(self, data): - metadata = self.metadata.get('application/xml', {}) - # We expect data to contain a single key which is the XML root. - root_key = data.keys()[0] - doc = minidom.Document() - node = self._to_xml_node(doc, metadata, root_key, data[root_key]) - - xmlns = node.getAttribute('xmlns') - if not xmlns and self.default_xmlns: - node.setAttribute('xmlns', self.default_xmlns) - - return node.toprettyxml(indent=' ') - - def _to_xml_node(self, doc, metadata, nodename, data): - """Recursive method to convert data members to XML nodes.""" - result = doc.createElement(nodename) - - # Set the xml namespace if one is specified - # TODO(justinsb): We could also use prefixes on the keys - xmlns = metadata.get('xmlns', None) - if xmlns: - result.setAttribute('xmlns', xmlns) - - if type(data) is list: - collections = metadata.get('list_collections', {}) - if nodename in collections: - metadata = collections[nodename] - for item in data: - node = doc.createElement(metadata['item_name']) - node.setAttribute(metadata['item_key'], str(item)) - result.appendChild(node) - return result - singular = metadata.get('plurals', {}).get(nodename, None) - if singular is None: - if nodename.endswith('s'): - singular = nodename[:-1] - else: - singular = 'item' - for item in data: - node = self._to_xml_node(doc, metadata, singular, item) - result.appendChild(node) - elif type(data) is dict: - collections = metadata.get('dict_collections', {}) - if nodename in collections: - metadata = collections[nodename] - for k, v in data.items(): - node = doc.createElement(metadata['item_name']) - node.setAttribute(metadata['item_key'], str(k)) - text = doc.createTextNode(str(v)) - node.appendChild(text) - result.appendChild(node) - return result - attrs = metadata.get('attributes', {}).get(nodename, {}) - for k, v in data.items(): - if k in attrs: - result.setAttribute(k, str(v)) - else: - node = self._to_xml_node(doc, metadata, k, v) - result.appendChild(node) - else: - # Type is atom - node = doc.createTextNode(str(data)) - result.appendChild(node) - return result - - def paste_config_file(basename): """Find the best location in the system for a paste config file. diff --git a/plugins/xenserver/networking/etc/init.d/openvswitch-nova b/plugins/xenserver/networking/etc/init.d/openvswitch-nova new file mode 100755 index 000000000..8672a69b8 --- /dev/null +++ b/plugins/xenserver/networking/etc/init.d/openvswitch-nova @@ -0,0 +1,96 @@ +#!/bin/bash +# +# openvswitch-nova +# +# chkconfig: 2345 96 89 +# description: Apply initial OVS flows for Nova + +# Copyright 2011 OpenStack LLC. +# Copyright (C) 2009, 2010, 2011 Nicira Networks, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# source function library +if [ -f /etc/init.d/functions ]; then + . /etc/init.d/functions +elif [ -f /etc/rc.d/init.d/functions ]; then + . /etc/rc.d/init.d/functions +elif [ -f /lib/lsb/init-functions ]; then + . /lib/lsb/init-functions +else + echo "$0: missing LSB shell function library" >&2 + exit 1 +fi + +OVS_CONFIGURE_BASE_FLOWS=/etc/xensource/scripts/ovs_configure_base_flows.py + +if test -e /etc/sysconfig/openvswitch-nova; then + . /etc/sysconfig/openvswitch-nova +else + echo "$0: missing configuration file: /etc/sysconfig/openvswitch-nova" + exit 1 +fi + +if test -e /etc/xensource/network.conf; then + NETWORK_MODE=$(cat /etc/xensource/network.conf) +fi + +case ${NETWORK_MODE:=openvswitch} in + vswitch|openvswitch) + ;; + bridge) + exit 0 + ;; + *) + echo "Open vSwitch disabled (/etc/xensource/network.conf is invalid)" >&2 + exit 0 + ;; +esac + +function run_ovs_conf_base_flows { + # expected format: DEVICE_BRIDGES="eth0:xenbr0 eth1:xenbr1" + for pair in $DEVICE_BRIDGES; do + # below in $info, physical device is [0], bridge name is [1] + info=${pair//:/ } + /usr/bin/python $OVS_CONFIGURE_BASE_FLOWS $1 ${info[0]} ${info[1]} + done +} + +function start { + run_ovs_conf_base_flows online +} + +function stop { + run_ovs_conf_base_flows offline +} + +function restart { + run_ovs_conf_base_flows reset +} + +case "$1" in + start) + start + ;; + stop) + stop + ;; + restart) + restart + ;; + *) + echo "usage: openvswitch-nova [start|stop|restart]" + exit 1 + ;; +esac diff --git a/plugins/xenserver/networking/etc/sysconfig/openvswitch-nova b/plugins/xenserver/networking/etc/sysconfig/openvswitch-nova new file mode 100644 index 000000000..829782fb6 --- /dev/null +++ b/plugins/xenserver/networking/etc/sysconfig/openvswitch-nova @@ -0,0 +1 @@ +#DEVICE_BRIDGES="eth0:xenbr0 eth1:xenbr1" diff --git a/plugins/xenserver/networking/etc/udev/rules.d/xen-openvswitch-nova.rules b/plugins/xenserver/networking/etc/udev/rules.d/xen-openvswitch-nova.rules new file mode 100644 index 000000000..b179f0847 --- /dev/null +++ b/plugins/xenserver/networking/etc/udev/rules.d/xen-openvswitch-nova.rules @@ -0,0 +1,3 @@ +SUBSYSTEM=="xen-backend", KERNEL=="vif*", RUN+="/etc/xensource/scripts/ovs_configure_vif_flows.py $env{ACTION} %k all" +# is this one needed? +#SUBSYSTEM=="net", KERNEL=="tap*", RUN+="/etc/xensource/scripts/ovs_configure_vif_flows.py $env{ACTION} %k all" diff --git a/plugins/xenserver/networking/etc/xensource/scripts/novalib.py b/plugins/xenserver/networking/etc/xensource/scripts/novalib.py new file mode 100644 index 000000000..dcbee3ded --- /dev/null +++ b/plugins/xenserver/networking/etc/xensource/scripts/novalib.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import os +import subprocess + + +def execute_get_output(*command): + """Execute and return stdout""" + devnull = open(os.devnull, 'w') + command = map(str, command) + proc = subprocess.Popen(command, close_fds=True, + stdout=subprocess.PIPE, stderr=devnull) + devnull.close() + return proc.stdout.read().strip() + + +def execute(*command): + """Execute without returning stdout""" + devnull = open(os.devnull, 'w') + command = map(str, command) + proc = subprocess.Popen(command, close_fds=True, + stdout=subprocess.PIPE, stderr=devnull) + devnull.close() diff --git a/plugins/xenserver/networking/etc/xensource/scripts/ovs_configure_base_flows.py b/plugins/xenserver/networking/etc/xensource/scripts/ovs_configure_base_flows.py new file mode 100755 index 000000000..514a43a2d --- /dev/null +++ b/plugins/xenserver/networking/etc/xensource/scripts/ovs_configure_base_flows.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +This script is used to configure base openvswitch flows for XenServer hosts. +""" + +import os +import sys + + +from novalib import execute, execute_get_output + + +def main(command, phys_dev_name, bridge_name): + ovs_ofctl = lambda *rule: execute('/usr/bin/ovs-ofctl', *rule) + + # always clear all flows first + ovs_ofctl('del-flows', bridge_name) + + if command in ('online', 'reset'): + pnic_ofport = execute_get_output('/usr/bin/ovs-vsctl', 'get', + 'Interface', phys_dev_name, 'ofport') + + # these flows are lower priority than all VM-specific flows. + + # allow all traffic from the physical NIC, as it is trusted (i.e., + # from a filtered vif, or from the physical infrastructure) + ovs_ofctl('add-flow', bridge_name, + "priority=2,in_port=%s,actions=normal" % pnic_ofport) + + # default drop + ovs_ofctl('add-flow', bridge_name, 'priority=1,actions=drop') + + +if __name__ == "__main__": + if len(sys.argv) != 4 or sys.argv[1] not in ('online', 'offline', 'reset'): + print sys.argv + script_name = os.path.basename(sys.argv[0]) + print "This script configures base ovs flows." + print "usage: %s [online|offline|reset] phys-dev-name bridge-name" \ + % script_name + print " ex: %s online eth0 xenbr0" % script_name + sys.exit(1) + else: + command, phys_dev_name, bridge_name = sys.argv[1:4] + main(command, phys_dev_name, bridge_name) diff --git a/plugins/xenserver/networking/etc/xensource/scripts/ovs_configure_vif_flows.py b/plugins/xenserver/networking/etc/xensource/scripts/ovs_configure_vif_flows.py new file mode 100755 index 000000000..accd08b91 --- /dev/null +++ b/plugins/xenserver/networking/etc/xensource/scripts/ovs_configure_vif_flows.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +This script is used to configure openvswitch flows on XenServer hosts. +""" + +import os +import sys + +# This is written to Python 2.4, since that is what is available on XenServer +import netaddr +import simplejson as json + +from novalib import execute, execute_get_output + + +OVS_OFCTL = '/usr/bin/ovs-ofctl' + + +class OvsFlow(object): + def __init__(self, bridge, params): + self.bridge = bridge + self.params = params + + def add(self, rule): + execute(OVS_OFCTL, 'add-flow', self.bridge, rule % self.params) + + def clear_flows(self, ofport): + execute(OVS_OFCTL, 'del-flows', self.bridge, "in_port=%s" % ofport) + + +def main(command, vif_raw, net_type): + if command not in ('online', 'offline'): + return + + vif_name, dom_id, vif_index = vif_raw.split('-') + vif = "%s%s.%s" % (vif_name, dom_id, vif_index) + bridge = "xenbr%s" % vif_index + + xsls = execute_get_output('/usr/bin/xenstore-ls', + '/local/domain/%s/vm-data/networking' % dom_id) + macs = [line.split("=")[0].strip() for line in xsls.splitlines()] + + for mac in macs: + xsread = execute_get_output('/usr/bin/xenstore-read', + '/local/domain/%s/vm-data/networking/%s' % + (dom_id, mac)) + data = json.loads(xsread) + if data["label"] == "public": + this_vif = "vif%s.0" % dom_id + else: + this_vif = "vif%s.1" % dom_id + + if vif == this_vif: + vif_ofport = execute_get_output('/usr/bin/ovs-vsctl', 'get', + 'Interface', vif, 'ofport') + + params = dict(VIF_NAME=vif, + MAC=data['mac'], + OF_PORT=vif_ofport) + + ovs = OvsFlow(bridge, params) + + if command == 'offline': + # I haven't found a way to clear only IPv4 or IPv6 rules. + ovs.clear_flows(vif_ofport) + + if command == 'online': + if net_type in ('ipv4', 'all') and 'ips' in data: + for ip4 in data['ips']: + ovs.params.update({'IPV4_ADDR': ip4['ip']}) + apply_ovs_ipv4_flows(ovs, bridge, params) + if net_type in ('ipv6', 'all') and 'ip6s' in data: + for ip6 in data['ip6s']: + link_local = str(netaddr.EUI(data['mac']).eui64()\ + .ipv6_link_local()) + ovs.params.update({'IPV6_LINK_LOCAL_ADDR': link_local}) + ovs.params.update({'IPV6_GLOBAL_ADDR': ip6['ip']}) + apply_ovs_ipv6_flows(ovs, bridge, params) + + +def apply_ovs_ipv4_flows(ovs, bridge, params): + # allow valid ARP outbound (both request / reply) + ovs.add("priority=3,in_port=%(OF_PORT)s,dl_src=%(MAC)s,arp," + "arp_sha=%(MAC)s,nw_src=%(IPV4_ADDR)s,actions=normal") + + ovs.add("priority=3,in_port=%(OF_PORT)s,dl_src=%(MAC)s,arp," + "arp_sha=%(MAC)s,nw_src=0.0.0.0,actions=normal") + + # allow valid IPv4 outbound + ovs.add("priority=3,in_port=%(OF_PORT)s,dl_src=%(MAC)s,ip," + "nw_src=%(IPV4_ADDR)s,actions=normal") + + +def apply_ovs_ipv6_flows(ovs, bridge, params): + # allow valid IPv6 ND outbound (are both global and local IPs needed?) + # Neighbor Solicitation + ovs.add("priority=6,in_port=%(OF_PORT)s,dl_src=%(MAC)s,icmp6," + "ipv6_src=%(IPV6_LINK_LOCAL_ADDR)s,icmp_type=135,nd_sll=%(MAC)s," + "actions=normal") + ovs.add("priority=6,in_port=%(OF_PORT)s,dl_src=%(MAC)s,icmp6," + "ipv6_src=%(IPV6_LINK_LOCAL_ADDR)s,icmp_type=135,actions=normal") + ovs.add("priority=6,in_port=%(OF_PORT)s,dl_src=%(MAC)s,icmp6," + "ipv6_src=%(IPV6_GLOBAL_ADDR)s,icmp_type=135,nd_sll=%(MAC)s," + "actions=normal") + ovs.add("priority=6,in_port=%(OF_PORT)s,dl_src=%(MAC)s,icmp6," + "ipv6_src=%(IPV6_GLOBAL_ADDR)s,icmp_type=135,actions=normal") + + # Neighbor Advertisement + ovs.add("priority=6,in_port=%(OF_PORT)s,dl_src=%(MAC)s,icmp6," + "ipv6_src=%(IPV6_LINK_LOCAL_ADDR)s,icmp_type=136," + "nd_target=%(IPV6_LINK_LOCAL_ADDR)s,actions=normal") + ovs.add("priority=6,in_port=%(OF_PORT)s,dl_src=%(MAC)s,icmp6," + "ipv6_src=%(IPV6_LINK_LOCAL_ADDR)s,icmp_type=136,actions=normal") + ovs.add("priority=6,in_port=%(OF_PORT)s,dl_src=%(MAC)s,icmp6," + "ipv6_src=%(IPV6_GLOBAL_ADDR)s,icmp_type=136," + "nd_target=%(IPV6_GLOBAL_ADDR)s,actions=normal") + ovs.add("priority=6,in_port=%(OF_PORT)s,dl_src=%(MAC)s,icmp6," + "ipv6_src=%(IPV6_GLOBAL_ADDR)s,icmp_type=136,actions=normal") + + # drop all other neighbor discovery (req b/c we permit all icmp6 below) + ovs.add("priority=5,in_port=%(OF_PORT)s,icmp6,icmp_type=135,actions=drop") + ovs.add("priority=5,in_port=%(OF_PORT)s,icmp6,icmp_type=136,actions=drop") + + # do not allow sending specifc ICMPv6 types + # Router Advertisement + ovs.add("priority=5,in_port=%(OF_PORT)s,icmp6,icmp_type=134,actions=drop") + # Redirect Gateway + ovs.add("priority=5,in_port=%(OF_PORT)s,icmp6,icmp_type=137,actions=drop") + # Mobile Prefix Solicitation + ovs.add("priority=5,in_port=%(OF_PORT)s,icmp6,icmp_type=146,actions=drop") + # Mobile Prefix Advertisement + ovs.add("priority=5,in_port=%(OF_PORT)s,icmp6,icmp_type=147,actions=drop") + # Multicast Router Advertisement + ovs.add("priority=5,in_port=%(OF_PORT)s,icmp6,icmp_type=151,actions=drop") + # Multicast Router Solicitation + ovs.add("priority=5,in_port=%(OF_PORT)s,icmp6,icmp_type=152,actions=drop") + # Multicast Router Termination + ovs.add("priority=5,in_port=%(OF_PORT)s,icmp6,icmp_type=153,actions=drop") + + # allow valid IPv6 outbound, by type + ovs.add("priority=4,in_port=%(OF_PORT)s,dl_src=%(MAC)s," + "ipv6_src=%(IPV6_GLOBAL_ADDR)s,icmp6,actions=normal") + ovs.add("priority=4,in_port=%(OF_PORT)s,dl_src=%(MAC)s," + "ipv6_src=%(IPV6_LINK_LOCAL_ADDR)s,icmp6,actions=normal") + ovs.add("priority=4,in_port=%(OF_PORT)s,dl_src=%(MAC)s," + "ipv6_src=%(IPV6_GLOBAL_ADDR)s,tcp6,actions=normal") + ovs.add("priority=4,in_port=%(OF_PORT)s,dl_src=%(MAC)s," + "ipv6_src=%(IPV6_LINK_LOCAL_ADDR)s,tcp6,actions=normal") + ovs.add("priority=4,in_port=%(OF_PORT)s,dl_src=%(MAC)s," + "ipv6_src=%(IPV6_GLOBAL_ADDR)s,udp6,actions=normal") + ovs.add("priority=4,in_port=%(OF_PORT)s,dl_src=%(MAC)s," + "ipv6_src=%(IPV6_LINK_LOCAL_ADDR)s,udp6,actions=normal") + # all else will be dropped ... + + +if __name__ == "__main__": + if len(sys.argv) != 4: + print "usage: %s [online|offline] vif-domid-idx [ipv4|ipv6|all] " % \ + os.path.basename(sys.argv[0]) + sys.exit(1) + else: + command, vif_raw, net_type = sys.argv[1:4] + main(command, vif_raw, net_type) diff --git a/plugins/xenserver/networking/etc/xensource/scripts/vif_5.6-fp1.patch b/plugins/xenserver/networking/etc/xensource/scripts/vif_5.6-fp1.patch index feaf1312d..d42a11eff 100644 --- a/plugins/xenserver/networking/etc/xensource/scripts/vif_5.6-fp1.patch +++ b/plugins/xenserver/networking/etc/xensource/scripts/vif_5.6-fp1.patch @@ -8,7 +8,7 @@ fi ;; -@@ -224,9 +225,11 @@ +@@ -224,6 +225,7 @@ remove) if [ "${TYPE}" = "vif" ] ;then @@ -16,7 +16,3 @@ xenstore-rm "${HOTPLUG}/hotplug" fi logger -t scripts-vif "${dev} has been removed" - remove_from_bridge - ;; - esac -+ diff --git a/plugins/xenserver/networking/etc/xensource/scripts/vif_rules.py b/plugins/xenserver/networking/etc/xensource/scripts/vif_rules.py index 48122e6d6..662def205 100755 --- a/plugins/xenserver/networking/etc/xensource/scripts/vif_rules.py +++ b/plugins/xenserver/networking/etc/xensource/scripts/vif_rules.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2010 OpenStack LLC. +# Copyright 2010-2011 OpenStack LLC. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -29,15 +29,18 @@ import sys import simplejson as json +from novalib import execute, execute_get_output + + def main(dom_id, command, only_this_vif=None): - xsls = execute('/usr/bin/xenstore-ls', - '/local/domain/%s/vm-data/networking' % dom_id, True) + xsls = execute_get_output('/usr/bin/xenstore-ls', + '/local/domain/%s/vm-data/networking' % dom_id) macs = [line.split("=")[0].strip() for line in xsls.splitlines()] for mac in macs: - xsread = execute('/usr/bin/enstore-read', - '/local/domain/%s/vm-data/networking/%s' % - (dom_id, mac), True) + xsread = execute_get_output('/usr/bin/xenstore-read', + '/local/domain/%s/vm-data/networking/%s' % + (dom_id, mac)) data = json.loads(xsread) for ip in data['ips']: if data["label"] == "public": @@ -52,17 +55,6 @@ def main(dom_id, command, only_this_vif=None): apply_iptables_rules(command, params) -def execute(*command, return_stdout=False): - devnull = open(os.devnull, 'w') - command = map(str, command) - proc = subprocess.Popen(command, close_fds=True, - stdout=subprocess.PIPE, stderr=devnull) - devnull.close() - if return_stdout: - return proc.stdout.read() - else: - return None - # A note about adding rules: # Whenever we add any rule to iptables, arptables or ebtables we first # delete the same rule to ensure the rule only exists once. @@ -113,8 +105,8 @@ def apply_ebtables_rules(command, params): ebtables('-D', 'FORWARD', '-p', '0806', '-o', params['VIF'], '--arp-ip-dst', params['IP'], '-j', 'ACCEPT') - ebtables('-D', 'FORWARD', '-p', '0800', '-o', - params['VIF'], '--ip-dst', params['IP'], + ebtables('-D', 'FORWARD', '-p', '0800', '-o', params['VIF'], + '--ip-dst', params['IP'], '-j', 'ACCEPT') if command == 'online': ebtables('-A', 'FORWARD', '-p', '0806', diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent index 9e761f264..b8a1b936a 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent @@ -53,6 +53,19 @@ class TimeoutError(StandardError): pass +def version(self, arg_dict): + """Get version of agent.""" + arg_dict["value"] = json.dumps({"name": "version", "value": ""}) + request_id = arg_dict["id"] + arg_dict["path"] = "data/host/%s" % request_id + xenstore.write_record(self, arg_dict) + try: + resp = _wait_for_agent(self, request_id, arg_dict) + except TimeoutError, e: + raise PluginError(e) + return resp + + def key_init(self, arg_dict): """Handles the Diffie-Hellman key exchange with the agent to establish the shared secret key used to encrypt/decrypt sensitive @@ -144,6 +157,23 @@ def inject_file(self, arg_dict): return resp +def agent_update(self, arg_dict): + """Expects an URL and md5sum of the contents, then directs the agent to + update itself.""" + request_id = arg_dict["id"] + url = arg_dict["url"] + md5sum = arg_dict["md5sum"] + arg_dict["value"] = json.dumps({"name": "agentupdate", + "value": {"url": url, "md5sum": md5sum}}) + arg_dict["path"] = "data/host/%s" % request_id + xenstore.write_record(self, arg_dict) + try: + resp = _wait_for_agent(self, request_id, arg_dict) + except TimeoutError, e: + raise PluginError(e) + return resp + + def _agent_has_method(self, method): """Check that the agent has a particular method by checking its features. Cache the features so we don't have to query the agent @@ -201,7 +231,9 @@ def _wait_for_agent(self, request_id, arg_dict): if __name__ == "__main__": XenAPIPlugin.dispatch( - {"key_init": key_init, + {"version": version, + "key_init": key_init, "password": password, "resetnetwork": resetnetwork, - "inject_file": inject_file}) + "inject_file": inject_file, + "agentupdate": agent_update}) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 4b45671ae..46031ebe8 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -22,6 +22,10 @@ # import httplib +try: + import json +except ImportError: + import simplejson as json import os import os.path import pickle @@ -87,8 +91,8 @@ def _download_tarball(sr_path, staging_path, image_id, glance_host, conn.close() -def _fixup_vhds(sr_path, staging_path, uuid_stack): - """Fixup the downloaded VHDs before we move them into the SR. +def _import_vhds(sr_path, staging_path, uuid_stack): + """Import the VHDs found in the staging path. We cannot extract VHDs directly into the SR since they don't yet have UUIDs, aren't properly associated with each other, and would be subject to @@ -98,16 +102,25 @@ def _fixup_vhds(sr_path, staging_path, uuid_stack): To avoid these we problems, we use a staging area to fixup the VHDs before moving them into the SR. The steps involved are: - 1. Extracting tarball into staging area + 1. Extracting tarball into staging area (done prior to this call) 2. Renaming VHDs to use UUIDs ('snap.vhd' -> 'ffff-aaaa-...vhd') - 3. Linking the two VHDs together + 3. Linking VHDs together if there's a snap.vhd 4. Pseudo-atomically moving the images into the SR. (It's not really - atomic because it takes place as two os.rename operations; however, - the chances of an SR.scan occuring between the two rename() + atomic because it takes place as multiple os.rename operations; + however, the chances of an SR.scan occuring between the rename()s invocations is so small that we can safely ignore it) + + Returns: A list of VDIs. Each list element is a dictionary containing + information about the VHD. Dictionary keys are: + 1. "vdi_type" - The type of VDI. Currently they can be "os_disk" or + "swap" + 2. "vdi_uuid" - The UUID of the VDI + + Example return: [{"vdi_type": "os_disk","vdi_uuid": "ffff-aaa..vhd"}, + {"vdi_type": "swap","vdi_uuid": "ffff-bbb..vhd"}] """ def rename_with_uuid(orig_path): """Rename VHD using UUID so that it will be recognized by SR on a @@ -158,27 +171,59 @@ def _fixup_vhds(sr_path, staging_path, uuid_stack): "VHD %(path)s is marked as hidden without child" % locals()) - orig_base_copy_path = os.path.join(staging_path, 'image.vhd') - if not os.path.exists(orig_base_copy_path): + def prepare_if_exists(staging_path, vhd_name, parent_path=None): + """ + Check for existance of a particular VHD in the staging path and + preparing it for moving into the SR. + + Returns: Tuple of (Path to move into the SR, VDI_UUID) + None, if the vhd_name doesn't exist in the staging path + + If the VHD exists, we will do the following: + 1. Rename it with a UUID. + 2. If parent_path exists, we'll link up the VHDs. + """ + orig_path = os.path.join(staging_path, vhd_name) + if not os.path.exists(orig_path): + return None + new_path, vdi_uuid = rename_with_uuid(orig_path) + if parent_path: + # NOTE(sirp): this step is necessary so that an SR scan won't + # delete the base_copy out from under us (since it would be + # orphaned) + link_vhds(new_path, parent_path) + return (new_path, vdi_uuid) + + vdi_return_list = [] + paths_to_move = [] + + image_info = prepare_if_exists(staging_path, 'image.vhd') + if not image_info: raise Exception("Invalid image: image.vhd not present") - base_copy_path, base_copy_uuid = rename_with_uuid(orig_base_copy_path) - - vdi_uuid = base_copy_uuid - orig_snap_path = os.path.join(staging_path, 'snap.vhd') - if os.path.exists(orig_snap_path): - snap_path, snap_uuid = rename_with_uuid(orig_snap_path) - vdi_uuid = snap_uuid - # NOTE(sirp): this step is necessary so that an SR scan won't - # delete the base_copy out from under us (since it would be - # orphaned) - link_vhds(snap_path, base_copy_path) - move_into_sr(snap_path) + paths_to_move.append(image_info[0]) + + snap_info = prepare_if_exists(staging_path, 'snap.vhd', + image_info[0]) + if snap_info: + paths_to_move.append(snap_info[0]) + # We return this snap as the VDI instead of image.vhd + vdi_return_list.append(dict(vdi_type="os", vdi_uuid=snap_info[1])) else: - assert_vhd_not_hidden(base_copy_path) + assert_vhd_not_hidden(image_info[0]) + # If there's no snap, we return the image.vhd UUID + vdi_return_list.append(dict(vdi_type="os", vdi_uuid=image_info[1])) + + swap_info = prepare_if_exists(staging_path, 'swap.vhd') + if swap_info: + assert_vhd_not_hidden(swap_info[0]) + paths_to_move.append(swap_info[0]) + vdi_return_list.append(dict(vdi_type="swap", vdi_uuid=swap_info[1])) + + for path in paths_to_move: + move_into_sr(path) - move_into_sr(base_copy_path) - return vdi_uuid + return vdi_return_list def _prepare_staging_area_for_upload(sr_path, staging_path, vdi_uuids): @@ -199,7 +244,7 @@ def _upload_tarball(staging_path, image_id, glance_host, glance_port, os_type): conn = httplib.HTTPConnection(glance_host, glance_port) # NOTE(sirp): httplib under python2.4 won't accept a file-like object # to request - conn.putrequest('PUT', '/images/%s' % image_id) + conn.putrequest('PUT', '/v1/images/%s' % image_id) # NOTE(sirp): There is some confusion around OVF. Here's a summary of # where we currently stand: @@ -324,8 +369,9 @@ def download_vhd(session, args): try: _download_tarball(sr_path, staging_path, image_id, glance_host, glance_port) - vdi_uuid = _fixup_vhds(sr_path, staging_path, uuid_stack) - return vdi_uuid + # Right now, it's easier to return a single string via XenAPI, + # so we'll json encode the list of VHDs. + return json.dumps(_import_vhds(sr_path, staging_path, uuid_stack)) finally: _cleanup_staging_area(staging_path) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index 75c653408..ac1c50ad9 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -44,7 +44,7 @@ def move_vhds_into_sr(session, args): new_cow_uuid = params['new_cow_uuid'] sr_path = params['sr_path'] - sr_temp_path = "%s/images/" % sr_path + sr_temp_path = "%s/tmp/" % sr_path # Discover the copied VHDs locally, and then set up paths to copy # them to under the SR diff --git a/run_tests.py b/run_tests.py index d5d8acd16..bb33f9139 100644 --- a/run_tests.py +++ b/run_tests.py @@ -56,9 +56,11 @@ To run a single test module: """ import gettext +import heapq import os import unittest import sys +import time gettext.install('nova', unicode=1) @@ -183,9 +185,21 @@ class _NullColorizer(object): self.stream.write(text) +def get_elapsed_time_color(elapsed_time): + if elapsed_time > 1.0: + return 'red' + elif elapsed_time > 0.25: + return 'yellow' + else: + return 'green' + + class NovaTestResult(result.TextTestResult): def __init__(self, *args, **kw): + self.show_elapsed = kw.pop('show_elapsed') result.TextTestResult.__init__(self, *args, **kw) + self.num_slow_tests = 5 + self.slow_tests = [] # this is a fixed-sized heap self._last_case = None self.colorizer = None # NOTE(vish): reset stdout for the terminal check @@ -197,28 +211,49 @@ class NovaTestResult(result.TextTestResult): break sys.stdout = stdout + # NOTE(lorinh): Initialize start_time in case a sqlalchemy-migrate + # error results in it failing to be initialized later. Otherwise, + # _handleElapsedTime will fail, causing the wrong error message to + # be outputted. + self.start_time = time.time() + def getDescription(self, test): return str(test) - # NOTE(vish): copied from unittest with edit to add color - def addSuccess(self, test): - unittest.TestResult.addSuccess(self, test) + def _handleElapsedTime(self, test): + self.elapsed_time = time.time() - self.start_time + item = (self.elapsed_time, test) + # Record only the n-slowest tests using heap + if len(self.slow_tests) >= self.num_slow_tests: + heapq.heappushpop(self.slow_tests, item) + else: + heapq.heappush(self.slow_tests, item) + + def _writeElapsedTime(self, test): + color = get_elapsed_time_color(self.elapsed_time) + self.colorizer.write(" %.2f" % self.elapsed_time, color) + + def _writeResult(self, test, long_result, color, short_result, success): if self.showAll: - self.colorizer.write("OK", 'green') + self.colorizer.write(long_result, color) + if self.show_elapsed and success: + self._writeElapsedTime(test) self.stream.writeln() elif self.dots: - self.stream.write('.') + self.stream.write(short_result) self.stream.flush() # NOTE(vish): copied from unittest with edit to add color + def addSuccess(self, test): + unittest.TestResult.addSuccess(self, test) + self._handleElapsedTime(test) + self._writeResult(test, 'OK', 'green', '.', True) + + # NOTE(vish): copied from unittest with edit to add color def addFailure(self, test, err): unittest.TestResult.addFailure(self, test, err) - if self.showAll: - self.colorizer.write("FAIL", 'red') - self.stream.writeln() - elif self.dots: - self.stream.write('F') - self.stream.flush() + self._handleElapsedTime(test) + self._writeResult(test, 'FAIL', 'red', 'F', False) # NOTE(vish): copied from nose with edit to add color def addError(self, test, err): @@ -226,6 +261,7 @@ class NovaTestResult(result.TextTestResult): errorClasses. If the exception is a registered class, the error will be added to the list for that class, not errors. """ + self._handleElapsedTime(test) stream = getattr(self, 'stream', None) ec, ev, tb = err try: @@ -252,14 +288,11 @@ class NovaTestResult(result.TextTestResult): self.errors.append((test, exc_info)) test.passed = False if stream is not None: - if self.showAll: - self.colorizer.write("ERROR", 'red') - self.stream.writeln() - elif self.dots: - stream.write('E') + self._writeResult(test, 'ERROR', 'red', 'E', False) def startTest(self, test): unittest.TestResult.startTest(self, test) + self.start_time = time.time() current_case = test.test.__class__.__name__ if self.showAll: @@ -273,21 +306,47 @@ class NovaTestResult(result.TextTestResult): class NovaTestRunner(core.TextTestRunner): + def __init__(self, *args, **kwargs): + self.show_elapsed = kwargs.pop('show_elapsed') + core.TextTestRunner.__init__(self, *args, **kwargs) + def _makeResult(self): return NovaTestResult(self.stream, self.descriptions, self.verbosity, - self.config) + self.config, + show_elapsed=self.show_elapsed) + + def _writeSlowTests(self, result_): + # Pare out 'fast' tests + slow_tests = [item for item in result_.slow_tests + if get_elapsed_time_color(item[0]) != 'green'] + if slow_tests: + slow_total_time = sum(item[0] for item in slow_tests) + self.stream.writeln("Slowest %i tests took %.2f secs:" + % (len(slow_tests), slow_total_time)) + for elapsed_time, test in sorted(slow_tests, reverse=True): + time_str = "%.2f" % elapsed_time + self.stream.writeln(" %s %s" % (time_str.ljust(10), test)) + + def run(self, test): + result_ = core.TextTestRunner.run(self, test) + if self.show_elapsed: + self._writeSlowTests(result_) + return result_ if __name__ == '__main__': logging.setup() # If any argument looks like a test name but doesn't have "nova.tests" in # front of it, automatically add that so we don't have to type as much + show_elapsed = True argv = [] for x in sys.argv: if x.startswith('test_'): argv.append('nova.tests.%s' % x) + elif x.startswith('--hide-elapsed'): + show_elapsed = False else: argv.append(x) @@ -300,5 +359,6 @@ if __name__ == '__main__': runner = NovaTestRunner(stream=c.stream, verbosity=c.verbosity, - config=c) + config=c, + show_elapsed=show_elapsed) sys.exit(not core.run(config=c, testRunner=runner, argv=argv)) diff --git a/run_tests.sh b/run_tests.sh index 9aa555484..c3f06f837 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -10,6 +10,7 @@ function usage { echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added." echo " -p, --pep8 Just run pep8" echo " -h, --help Print this usage message" + echo " --hide-elapsed Don't print the elapsed time for each test along with slow test list" echo "" echo "Note: with no options specified, the script will try to run the tests in a virtual environment," echo " If no virtualenv is found, the script will ask if you would like to create one. If you " @@ -24,6 +25,7 @@ function process_option { -N|--no-virtual-env) let always_venv=0; let never_venv=1;; -f|--force) let force=1;; -p|--pep8) let just_pep8=1;; + -*) noseopts="$noseopts $1";; *) noseargs="$noseargs $1" esac } @@ -34,6 +36,7 @@ always_venv=0 never_venv=0 force=0 noseargs= +noseopts= wrapper="" just_pep8=0 @@ -67,15 +70,12 @@ function run_pep8 { srcfiles=`find bin -type f ! -name "nova.conf*"` srcfiles+=" `find tools/*`" srcfiles+=" nova setup.py plugins/xenserver/xenapi/etc/xapi.d/plugins/glance" - pep8 --repeat --show-pep8 --show-source --exclude=vcsversion.py ${srcfiles} + # Just run PEP8 in current environment + ${wrapper} pep8 --repeat --show-pep8 --show-source \ + --exclude=vcsversion.py ${srcfiles} } -if [ $just_pep8 -eq 1 ]; then - run_pep8 - exit -fi - -NOSETESTS="python run_tests.py $noseargs" +NOSETESTS="python run_tests.py $noseopts $noseargs" if [ $never_venv -eq 0 ] then @@ -103,9 +103,17 @@ then fi fi +if [ $just_pep8 -eq 1 ]; then + run_pep8 + exit +fi + run_tests || exit -# Also run pep8 if no options were provided. +# NOTE(sirp): we only want to run pep8 when we're running the full-test suite, +# not when we're running tests individually. To handle this, we need to +# distinguish between options (noseopts), which begin with a '-', and +# arguments (noseargs). if [ -z "$noseargs" ]; then run_pep8 fi diff --git a/tools/install_venv.py b/tools/install_venv.py index 812b1dd0f..f4b6583ed 100644 --- a/tools/install_venv.py +++ b/tools/install_venv.py @@ -36,7 +36,7 @@ PY_VERSION = "python%s.%s" % (sys.version_info[0], sys.version_info[1]) def die(message, *args): - print >>sys.stderr, message % args + print >> sys.stderr, message % args sys.exit(1) diff --git a/tools/pip-requires b/tools/pip-requires index 8f8018765..5d31a814d 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -1,7 +1,6 @@ SQLAlchemy==0.6.3 pep8==0.5.0 pylint==0.19 -IPy==0.70 Cheetah==2.4.4 M2Crypto==0.20.2 amqplib==0.6.1 @@ -10,15 +9,14 @@ boto==1.9b carrot==0.10.5 eventlet==0.9.12 lockfile==0.8 -python-novaclient==2.3 +python-novaclient==2.5.3 python-daemon==1.5.5 python-gflags==1.3 redis==2.0.0 routes==1.12.3 -WebOb==0.9.8 +WebOb==1.0.8 wsgiref==0.1.2 -mox==0.5.0 --f http://pymox.googlecode.com/files/mox-0.5.0.tar.gz +mox==0.5.3 greenlet==0.3.1 nose bzr @@ -34,3 +32,4 @@ suds==0.4 coverage nosexcover GitPython +paramiko |
