diff options
| author | Chuck Short <zulcss@ubuntu.com> | 2011-06-23 09:54:54 -0400 |
|---|---|---|
| committer | Chuck Short <zulcss@ubuntu.com> | 2011-06-23 09:54:54 -0400 |
| commit | 3b120fe30c0a9999b7ba85e07a262e672a1c8ed2 (patch) | |
| tree | cf7127582f31a37b2a0beb4f6c8b2ad304e64086 | |
| parent | a2cf08c84bc9c4113911534cfe3d4460b56d4124 (diff) | |
| parent | 6d6720e9b7e52461238ece684c9acc7183673bb8 (diff) | |
| download | nova-3b120fe30c0a9999b7ba85e07a262e672a1c8ed2.tar.gz nova-3b120fe30c0a9999b7ba85e07a262e672a1c8ed2.tar.xz nova-3b120fe30c0a9999b7ba85e07a262e672a1c8ed2.zip | |
Merge trunk:
87 files changed, 3139 insertions, 1309 deletions
diff --git a/bin/nova-manage b/bin/nova-manage index 4261a4a3a..ba1436916 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -97,7 +97,6 @@ flags.DECLARE('vlan_start', 'nova.network.manager') flags.DECLARE('vpn_start', 'nova.network.manager') flags.DECLARE('fixed_range_v6', 'nova.network.manager') flags.DECLARE('gateway_v6', 'nova.network.manager') -flags.DECLARE('images_path', 'nova.image.local') flags.DECLARE('libvirt_type', 'nova.virt.libvirt.connection') flags.DEFINE_flag(flags.HelpFlag()) flags.DEFINE_flag(flags.HelpshortFlag()) @@ -874,7 +873,7 @@ class InstanceTypeCommands(object): try: instance_types.create(name, memory, vcpus, local_gb, flavorid, swap, rxtx_quota, rxtx_cap) - except exception.InvalidInputException: + except exception.InvalidInput: print "Must supply valid parameters to create instance_type" print e sys.exit(1) @@ -1056,16 +1055,6 @@ class ImageCommands(object): machine_images = {} other_images = {} directory = os.path.abspath(directory) - # NOTE(vish): If we're importing from the images path dir, attempt - # to move the files out of the way before importing - # so we aren't writing to the same directory. This - # may fail if the dir was a mointpoint. - if (FLAGS.image_service == 'nova.image.local.LocalImageService' - and directory == os.path.abspath(FLAGS.images_path)): - new_dir = "%s_bak" % directory - os.rename(directory, new_dir) - os.mkdir(directory) - directory = new_dir for fn in glob.glob("%s/*/info.json" % directory): try: image_path = os.path.join(fn.rpartition('/')[0], 'image') @@ -1082,6 +1071,70 @@ class ImageCommands(object): self._convert_images(machine_images) +class AgentBuildCommands(object): + """Class for managing agent builds.""" + + def create(self, os, architecture, version, url, md5hash, + hypervisor='xen'): + """Creates a new agent build. + arguments: os architecture version url md5hash [hypervisor='xen']""" + ctxt = context.get_admin_context() + agent_build = db.agent_build_create(ctxt, + {'hypervisor': hypervisor, + 'os': os, + 'architecture': architecture, + 'version': version, + 'url': url, + 'md5hash': md5hash}) + + def delete(self, os, architecture, hypervisor='xen'): + """Deletes an existing agent build. + arguments: os architecture [hypervisor='xen']""" + ctxt = context.get_admin_context() + agent_build_ref = db.agent_build_get_by_triple(ctxt, + hypervisor, os, architecture) + db.agent_build_destroy(ctxt, agent_build_ref['id']) + + def list(self, hypervisor=None): + """Lists all agent builds. + arguments: <none>""" + fmt = "%-10s %-8s %12s %s" + ctxt = context.get_admin_context() + by_hypervisor = {} + for agent_build in db.agent_build_get_all(ctxt): + buildlist = by_hypervisor.get(agent_build.hypervisor) + if not buildlist: + buildlist = by_hypervisor[agent_build.hypervisor] = [] + + buildlist.append(agent_build) + + for key, buildlist in by_hypervisor.iteritems(): + if hypervisor and key != hypervisor: + continue + + print "Hypervisor: %s" % key + print fmt % ('-' * 10, '-' * 8, '-' * 12, '-' * 32) + for agent_build in buildlist: + print fmt % (agent_build.os, agent_build.architecture, + agent_build.version, agent_build.md5hash) + print ' %s' % agent_build.url + + print + + def modify(self, os, architecture, version, url, md5hash, + hypervisor='xen'): + """Update an existing agent build. + arguments: os architecture version url md5hash [hypervisor='xen'] + """ + ctxt = context.get_admin_context() + agent_build_ref = db.agent_build_get_by_triple(ctxt, + hypervisor, os, architecture) + db.agent_build_update(ctxt, agent_build_ref['id'], + {'version': version, + 'url': url, + 'md5hash': md5hash}) + + class ConfigCommands(object): """Class for exposing the flags defined by flag_file(s).""" @@ -1094,6 +1147,7 @@ class ConfigCommands(object): CATEGORIES = [ ('account', AccountCommands), + ('agent', AgentBuildCommands), ('config', ConfigCommands), ('db', DbCommands), ('fixed', FixedIpCommands), diff --git a/doc/.autogenerated b/doc/.autogenerated deleted file mode 100644 index 456c8ad1e..000000000 --- a/doc/.autogenerated +++ /dev/null @@ -1,283 +0,0 @@ -source/api/nova..adminclient.rst -source/api/nova..api.direct.rst -source/api/nova..api.ec2.admin.rst -source/api/nova..api.ec2.apirequest.rst -source/api/nova..api.ec2.cloud.rst -source/api/nova..api.ec2.metadatarequesthandler.rst -source/api/nova..api.openstack.auth.rst -source/api/nova..api.openstack.backup_schedules.rst -source/api/nova..api.openstack.common.rst -source/api/nova..api.openstack.consoles.rst -source/api/nova..api.openstack.faults.rst -source/api/nova..api.openstack.flavors.rst -source/api/nova..api.openstack.images.rst -source/api/nova..api.openstack.servers.rst -source/api/nova..api.openstack.shared_ip_groups.rst -source/api/nova..api.openstack.zones.rst -source/api/nova..auth.dbdriver.rst -source/api/nova..auth.fakeldap.rst -source/api/nova..auth.ldapdriver.rst -source/api/nova..auth.manager.rst -source/api/nova..auth.signer.rst -source/api/nova..cloudpipe.pipelib.rst -source/api/nova..compute.api.rst -source/api/nova..compute.instance_types.rst -source/api/nova..compute.manager.rst -source/api/nova..compute.monitor.rst -source/api/nova..compute.power_state.rst -source/api/nova..console.api.rst -source/api/nova..console.fake.rst -source/api/nova..console.manager.rst -source/api/nova..console.xvp.rst -source/api/nova..context.rst -source/api/nova..crypto.rst -source/api/nova..db.api.rst -source/api/nova..db.base.rst -source/api/nova..db.migration.rst -source/api/nova..db.sqlalchemy.api.rst -source/api/nova..db.sqlalchemy.migrate_repo.manage.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.005_add_instance_metadata.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.006_add_provider_data_to_volumes.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.007_add_instance_types.rst -source/api/nova..db.sqlalchemy.migration.rst -source/api/nova..db.sqlalchemy.models.rst -source/api/nova..db.sqlalchemy.session.rst -source/api/nova..exception.rst -source/api/nova..fakememcache.rst -source/api/nova..fakerabbit.rst -source/api/nova..flags.rst -source/api/nova..image.glance.rst -source/api/nova..image.local.rst -source/api/nova..image.s3.rst -source/api/nova..image.service.rst -source/api/nova..log.rst -source/api/nova..manager.rst -source/api/nova..network.api.rst -source/api/nova..network.linux_net.rst -source/api/nova..network.manager.rst -source/api/nova..objectstore.bucket.rst -source/api/nova..objectstore.handler.rst -source/api/nova..objectstore.image.rst -source/api/nova..objectstore.stored.rst -source/api/nova..quota.rst -source/api/nova..rpc.rst -source/api/nova..scheduler.chance.rst -source/api/nova..scheduler.driver.rst -source/api/nova..scheduler.manager.rst -source/api/nova..scheduler.simple.rst -source/api/nova..scheduler.zone.rst -source/api/nova..service.rst -source/api/nova..test.rst -source/api/nova..tests.api.openstack.fakes.rst -source/api/nova..tests.api.openstack.test_adminapi.rst -source/api/nova..tests.api.openstack.test_api.rst -source/api/nova..tests.api.openstack.test_auth.rst -source/api/nova..tests.api.openstack.test_common.rst -source/api/nova..tests.api.openstack.test_faults.rst -source/api/nova..tests.api.openstack.test_flavors.rst -source/api/nova..tests.api.openstack.test_images.rst -source/api/nova..tests.api.openstack.test_ratelimiting.rst -source/api/nova..tests.api.openstack.test_servers.rst -source/api/nova..tests.api.openstack.test_shared_ip_groups.rst -source/api/nova..tests.api.openstack.test_zones.rst -source/api/nova..tests.api.test_wsgi.rst -source/api/nova..tests.db.fakes.rst -source/api/nova..tests.declare_flags.rst -source/api/nova..tests.fake_flags.rst -source/api/nova..tests.glance.stubs.rst -source/api/nova..tests.hyperv_unittest.rst -source/api/nova..tests.objectstore_unittest.rst -source/api/nova..tests.real_flags.rst -source/api/nova..tests.runtime_flags.rst -source/api/nova..tests.test_access.rst -source/api/nova..tests.test_api.rst -source/api/nova..tests.test_auth.rst -source/api/nova..tests.test_cloud.rst -source/api/nova..tests.test_compute.rst -source/api/nova..tests.test_console.rst -source/api/nova..tests.test_direct.rst -source/api/nova..tests.test_flags.rst -source/api/nova..tests.test_instance_types.rst -source/api/nova..tests.test_localization.rst -source/api/nova..tests.test_log.rst -source/api/nova..tests.test_middleware.rst -source/api/nova..tests.test_misc.rst -source/api/nova..tests.test_network.rst -source/api/nova..tests.test_quota.rst -source/api/nova..tests.test_rpc.rst -source/api/nova..tests.test_scheduler.rst -source/api/nova..tests.test_service.rst -source/api/nova..tests.test_test.rst -source/api/nova..tests.test_twistd.rst -source/api/nova..tests.test_utils.rst -source/api/nova..tests.test_virt.rst -source/api/nova..tests.test_volume.rst -source/api/nova..tests.test_xenapi.rst -source/api/nova..tests.xenapi.stubs.rst -source/api/nova..twistd.rst -source/api/nova..utils.rst -source/api/nova..version.rst -source/api/nova..virt.connection.rst -source/api/nova..virt.disk.rst -source/api/nova..virt.fake.rst -source/api/nova..virt.hyperv.rst -source/api/nova..virt.images.rst -source/api/nova..virt.libvirt_conn.rst -source/api/nova..virt.xenapi.fake.rst -source/api/nova..virt.xenapi.network_utils.rst -source/api/nova..virt.xenapi.vm_utils.rst -source/api/nova..virt.xenapi.vmops.rst -source/api/nova..virt.xenapi.volume_utils.rst -source/api/nova..virt.xenapi.volumeops.rst -source/api/nova..virt.xenapi_conn.rst -source/api/nova..volume.api.rst -source/api/nova..volume.driver.rst -source/api/nova..volume.manager.rst -source/api/nova..volume.san.rst -source/api/nova..wsgi.rst -source/api/autoindex.rst -source/api/nova..adminclient.rst -source/api/nova..api.direct.rst -source/api/nova..api.ec2.admin.rst -source/api/nova..api.ec2.apirequest.rst -source/api/nova..api.ec2.cloud.rst -source/api/nova..api.ec2.metadatarequesthandler.rst -source/api/nova..api.openstack.auth.rst -source/api/nova..api.openstack.backup_schedules.rst -source/api/nova..api.openstack.common.rst -source/api/nova..api.openstack.consoles.rst -source/api/nova..api.openstack.faults.rst -source/api/nova..api.openstack.flavors.rst -source/api/nova..api.openstack.images.rst -source/api/nova..api.openstack.servers.rst -source/api/nova..api.openstack.shared_ip_groups.rst -source/api/nova..api.openstack.zones.rst -source/api/nova..auth.dbdriver.rst -source/api/nova..auth.fakeldap.rst -source/api/nova..auth.ldapdriver.rst -source/api/nova..auth.manager.rst -source/api/nova..auth.signer.rst -source/api/nova..cloudpipe.pipelib.rst -source/api/nova..compute.api.rst -source/api/nova..compute.instance_types.rst -source/api/nova..compute.manager.rst -source/api/nova..compute.monitor.rst -source/api/nova..compute.power_state.rst -source/api/nova..console.api.rst -source/api/nova..console.fake.rst -source/api/nova..console.manager.rst -source/api/nova..console.xvp.rst -source/api/nova..context.rst -source/api/nova..crypto.rst -source/api/nova..db.api.rst -source/api/nova..db.base.rst -source/api/nova..db.migration.rst -source/api/nova..db.sqlalchemy.api.rst -source/api/nova..db.sqlalchemy.migrate_repo.manage.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.001_austin.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.002_bexar.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.003_add_label_to_networks.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.004_add_zone_tables.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.005_add_instance_metadata.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.006_add_provider_data_to_volumes.rst -source/api/nova..db.sqlalchemy.migrate_repo.versions.007_add_instance_types.rst -source/api/nova..db.sqlalchemy.migration.rst -source/api/nova..db.sqlalchemy.models.rst -source/api/nova..db.sqlalchemy.session.rst -source/api/nova..exception.rst -source/api/nova..fakememcache.rst -source/api/nova..fakerabbit.rst -source/api/nova..flags.rst -source/api/nova..image.glance.rst -source/api/nova..image.local.rst -source/api/nova..image.s3.rst -source/api/nova..image.service.rst -source/api/nova..log.rst -source/api/nova..manager.rst -source/api/nova..network.api.rst -source/api/nova..network.linux_net.rst -source/api/nova..network.manager.rst -source/api/nova..objectstore.bucket.rst -source/api/nova..objectstore.handler.rst -source/api/nova..objectstore.image.rst -source/api/nova..objectstore.stored.rst -source/api/nova..quota.rst -source/api/nova..rpc.rst -source/api/nova..scheduler.chance.rst -source/api/nova..scheduler.driver.rst -source/api/nova..scheduler.manager.rst -source/api/nova..scheduler.simple.rst -source/api/nova..scheduler.zone.rst -source/api/nova..service.rst -source/api/nova..test.rst -source/api/nova..tests.api.openstack.fakes.rst -source/api/nova..tests.api.openstack.test_adminapi.rst -source/api/nova..tests.api.openstack.test_api.rst -source/api/nova..tests.api.openstack.test_auth.rst -source/api/nova..tests.api.openstack.test_common.rst -source/api/nova..tests.api.openstack.test_faults.rst -source/api/nova..tests.api.openstack.test_flavors.rst -source/api/nova..tests.api.openstack.test_images.rst -source/api/nova..tests.api.openstack.test_ratelimiting.rst -source/api/nova..tests.api.openstack.test_servers.rst -source/api/nova..tests.api.openstack.test_shared_ip_groups.rst -source/api/nova..tests.api.openstack.test_zones.rst -source/api/nova..tests.api.test_wsgi.rst -source/api/nova..tests.db.fakes.rst -source/api/nova..tests.declare_flags.rst -source/api/nova..tests.fake_flags.rst -source/api/nova..tests.glance.stubs.rst -source/api/nova..tests.hyperv_unittest.rst -source/api/nova..tests.objectstore_unittest.rst -source/api/nova..tests.real_flags.rst -source/api/nova..tests.runtime_flags.rst -source/api/nova..tests.test_access.rst -source/api/nova..tests.test_api.rst -source/api/nova..tests.test_auth.rst -source/api/nova..tests.test_cloud.rst -source/api/nova..tests.test_compute.rst -source/api/nova..tests.test_console.rst -source/api/nova..tests.test_direct.rst -source/api/nova..tests.test_flags.rst -source/api/nova..tests.test_instance_types.rst -source/api/nova..tests.test_localization.rst -source/api/nova..tests.test_log.rst -source/api/nova..tests.test_middleware.rst -source/api/nova..tests.test_misc.rst -source/api/nova..tests.test_network.rst -source/api/nova..tests.test_quota.rst -source/api/nova..tests.test_rpc.rst -source/api/nova..tests.test_scheduler.rst -source/api/nova..tests.test_service.rst -source/api/nova..tests.test_test.rst -source/api/nova..tests.test_twistd.rst -source/api/nova..tests.test_utils.rst -source/api/nova..tests.test_virt.rst -source/api/nova..tests.test_volume.rst -source/api/nova..tests.test_xenapi.rst -source/api/nova..tests.xenapi.stubs.rst -source/api/nova..twistd.rst -source/api/nova..utils.rst -source/api/nova..version.rst -source/api/nova..virt.connection.rst -source/api/nova..virt.disk.rst -source/api/nova..virt.fake.rst -source/api/nova..virt.hyperv.rst -source/api/nova..virt.images.rst -source/api/nova..virt.libvirt_conn.rst -source/api/nova..virt.xenapi.fake.rst -source/api/nova..virt.xenapi.network_utils.rst -source/api/nova..virt.xenapi.vm_utils.rst -source/api/nova..virt.xenapi.vmops.rst -source/api/nova..virt.xenapi.volume_utils.rst -source/api/nova..virt.xenapi.volumeops.rst -source/api/nova..virt.xenapi_conn.rst -source/api/nova..volume.api.rst -source/api/nova..volume.driver.rst -source/api/nova..volume.manager.rst -source/api/nova..volume.san.rst -source/api/nova..wsgi.rst diff --git a/doc/source/devref/distributed_scheduler.rst b/doc/source/devref/distributed_scheduler.rst index eb6a1a03e..e33fda4d2 100644 --- a/doc/source/devref/distributed_scheduler.rst +++ b/doc/source/devref/distributed_scheduler.rst @@ -14,10 +14,16 @@ License for the specific language governing permissions and limitations under the License. + Source for illustrations in doc/source/image_src/zone_distsched_illustrations.odp + (OpenOffice Impress format) Illustrations are "exported" to png and then scaled + to 400x300 or 640x480 as needed and placed in the doc/source/images directory. + Distributed Scheduler -===== +===================== -The Scheduler is akin to a Dating Service. Requests for the creation of new instances come in and the most applicable Compute nodes are selected from a large pool of potential candidates. In a small deployment we may be happy with the currently available Change Scheduler which randomly selects a Host from the available pool. Or if you need something a little more fancy you may want to use the Availability Zone Scheduler, which selects Compute hosts from a logical partitioning of available hosts (within a single Zone). +The Scheduler is akin to a Dating Service. Requests for the creation of new instances come in and the most applicable Compute nodes are selected from a large pool of potential candidates. In a small deployment we may be happy with the currently available Chance Scheduler which randomly selects a Host from the available pool. Or if you need something a little more fancy you may want to use the Availability Zone Scheduler, which selects Compute hosts from a logical partitioning of available hosts (within a single Zone). + + .. image:: /images/dating_service.png But for larger deployments a more complex scheduling algorithm is required. Additionally, if you are using Zones in your Nova setup, you'll need a scheduler that understand how to pass instance requests from Zone to Zone. @@ -25,75 +31,87 @@ This is the purpose of the Distributed Scheduler (DS). The DS utilizes the Capab So, how does this all work? -This document will explain the strategy employed by the `ZoneAwareScheduler` and its derivations. You should read the Zones documentation before reading this. +This document will explain the strategy employed by the `ZoneAwareScheduler` and its derivations. You should read the :doc:`devguide/zones` documentation before reading this. + + .. image:: /images/zone_aware_scheduler.png Costs & Weights ----------- -When deciding where to place an Instance, we compare a Weighted Cost for each Host. The Weighting, currently, is just the sum of each Cost. Costs are nothing more than integers from `0 - max_int`. Costs are computed by looking at the various Capabilities of the Host relative to the specs of the Instance being asked for. Trying to putting a plain vanilla instance on a high performance host should have a very high cost. But putting a vanilla instance on a vanilla Host should have a low cost. +--------------- +When deciding where to place an Instance, we compare a Weighted Cost for each Host. The Weighting, currently, is just the sum of each Cost. Costs are nothing more than integers from `0 - max_int`. Costs are computed by looking at the various Capabilities of the Host relative to the specs of the Instance being asked for. Trying to put a plain vanilla instance on a high performance host should have a very high cost. But putting a vanilla instance on a vanilla Host should have a low cost. Some Costs are more esoteric. Consider a rule that says we should prefer Hosts that don't already have an instance on it that is owned by the user requesting it (to mitigate against machine failures). Here we have to look at all the other Instances on the host to compute our cost. An example of some other costs might include selecting: -* a GPU-based host over a standard CPU -* a host with fast ethernet over a 10mbps line -* a host that can run Windows instances -* a host in the EU vs North America -* etc + * a GPU-based host over a standard CPU + * a host with fast ethernet over a 10mbps line + * a host that can run Windows instances + * a host in the EU vs North America + * etc This Weight is computed for each Instance requested. If the customer asked for 1000 instances, the consumed resources on each Host are "virtually" depleted so the Cost can change accordingly. + .. image:: /images/costs_weights.png + nova.scheduler.zone_aware_scheduler.ZoneAwareScheduler ------------ +------------------------------------------------------ As we explained in the Zones documentation, each Scheduler has a `ZoneManager` object that collects "Capabilities" about child Zones and each of the services running in the current Zone. The `ZoneAwareScheduler` uses this information to make its decisions. Here is how it works: -1. The compute nodes are filtered and the nodes remaining are weighed. -1a. Filtering the hosts is a simple matter of ensuring the compute node has ample resources (CPU, RAM, Disk, etc) to fulfil the request. -1b. Weighing of the remaining compute nodes assigns a number based on their suitability for the request. -2. The same request is sent to each child Zone and step #1 is done there too. The resulting weighted list is returned to the parent. -3. The parent Zone sorts and aggregates all the weights and a final build plan is constructed. -4. The build plan is executed upon. Concurrently, instance create requests are sent to each of the selected hosts, be they local or in a child zone. Child Zones may forward the requests to their child Zones as needed. + 1. The compute nodes are filtered and the nodes remaining are weighed. + 2. Filtering the hosts is a simple matter of ensuring the compute node has ample resources (CPU, RAM, Disk, etc) to fulfil the request. + 3. Weighing of the remaining compute nodes assigns a number based on their suitability for the request. + 4. The same request is sent to each child Zone and step #1 is done there too. The resulting weighted list is returned to the parent. + 5. The parent Zone sorts and aggregates all the weights and a final build plan is constructed. + 6. The build plan is executed upon. Concurrently, instance create requests are sent to each of the selected hosts, be they local or in a child zone. Child Zones may forward the requests to their child Zones as needed. + + .. image:: /images/zone_aware_overview.png `ZoneAwareScheduler` by itself is not capable of handling all the provisioning itself. Derived classes are used to select which host filtering and weighing strategy will be used. Filtering and Weighing ------------- +---------------------- The filtering (excluding compute nodes incapable of fulfilling the request) and weighing (computing the relative "fitness" of a compute node to fulfill the request) rules used are very subjective operations ... Service Providers will probably have a very different set of filtering and weighing rules than private cloud administrators. The filtering and weighing aspects of the `ZoneAwareScheduler` are flexible and extensible. + .. image:: /images/filtering.png + Requesting a new instance ------------- +------------------------- Prior to the `ZoneAwareScheduler`, to request a new instance, a call was made to `nova.compute.api.create()`. The type of instance created depended on the value of the `InstanceType` record being passed in. The `InstanceType` determined the amount of disk, CPU, RAM and network required for the instance. Administrators can add new `InstanceType` records to suit their needs. For more complicated instance requests we need to go beyond the default fields in the `InstanceType` table. `nova.compute.api.create()` performed the following actions: -1. it validated all the fields passed into it. -2. it created an entry in the `Instance` table for each instance requested -3. it put one `run_instance` message in the scheduler queue for each instance requested -4. the schedulers picked off the messages and decided which compute node should handle the request. -5. the `run_instance` message was forwarded to the compute node for processing and the instance is created. -6. it returned a list of dicts representing each of the `Instance` records (even if the instance has not been activated yet). At least the `instance_id`s are valid. + 1. it validated all the fields passed into it. + 2. it created an entry in the `Instance` table for each instance requested + 3. it put one `run_instance` message in the scheduler queue for each instance requested + 4. the schedulers picked off the messages and decided which compute node should handle the request. + 5. the `run_instance` message was forwarded to the compute node for processing and the instance is created. + 6. it returned a list of dicts representing each of the `Instance` records (even if the instance has not been activated yet). At least the `instance_ids` are valid. + + .. image:: /images/nova.compute.api.create.png Generally, the standard schedulers (like `ChanceScheduler` and `AvailabilityZoneScheduler`) only operate in the current Zone. They have no concept of child Zones. The problem with this approach is each request is scattered amongst each of the schedulers. If we are asking for 1000 instances, each scheduler gets the requests one-at-a-time. There is no possability of optimizing the requests to take into account all 1000 instances as a group. We call this Single-Shot vs. All-at-Once. For the `ZoneAwareScheduler` we need to use the All-at-Once approach. We need to consider all the hosts across all the Zones before deciding where they should reside. In order to handle this we have a new method `nova.compute.api.create_all_at_once()`. This method does things a little differently: -1. it validates all the fields passed into it. -2. it creates a single `reservation_id` for all of instances created. This is a UUID. -3. it creates a single `run_instance` request in the scheduler queue -4. a scheduler picks the message off the queue and works on it. -5. the scheduler sends off an OS API `POST /zones/select` command to each child Zone. The `BODY` payload of the call contains the `request_spec`. -6. the child Zones use the `request_spec` to compute a weighted list for each instance requested. No attempt to actually create an instance is done at this point. We're only estimating the suitability of the Zones. -7. if the child Zone has its own child Zones, the `/zones/select` call will be sent down to them as well. -8. Finally, when all the estimates have bubbled back to the Zone that initiated the call, all the results are merged, sorted and processed. -9. Now the instances can be created. The initiating Zone either forwards the `run_instance` message to the local Compute node to do the work, or it issues a `POST /servers` call to the relevant child Zone. The parameters to the child Zone call are the same as what was passed in by the user. -10. The `reservation_id` is passed back to the caller. Later we explain how the user can check on the status of the command with this `reservation_id`. + 1. it validates all the fields passed into it. + 2. it creates a single `reservation_id` for all of instances created. This is a UUID. + 3. it creates a single `run_instance` request in the scheduler queue + 4. a scheduler picks the message off the queue and works on it. + 5. the scheduler sends off an OS API `POST /zones/select` command to each child Zone. The `BODY` payload of the call contains the `request_spec`. + 6. the child Zones use the `request_spec` to compute a weighted list for each instance requested. No attempt to actually create an instance is done at this point. We're only estimating the suitability of the Zones. + 7. if the child Zone has its own child Zones, the `/zones/select` call will be sent down to them as well. + 8. Finally, when all the estimates have bubbled back to the Zone that initiated the call, all the results are merged, sorted and processed. + 9. Now the instances can be created. The initiating Zone either forwards the `run_instance` message to the local Compute node to do the work, or it issues a `POST /servers` call to the relevant child Zone. The parameters to the child Zone call are the same as what was passed in by the user. + 10. The `reservation_id` is passed back to the caller. Later we explain how the user can check on the status of the command with this `reservation_id`. + + .. image:: /images/nova.compute.api.create_all_at_once.png The Catch -------------- +--------- This all seems pretty straightforward but, like most things, there's a catch. Zones are expected to operate in complete isolation from each other. Each Zone has its own AMQP service, database and set of Nova services. But, for security reasons Zones should never leak information about the architectural layout internally. That means Zones cannot leak information about hostnames or service IP addresses outside of its world. -When `POST /zones/select` is called to estimate which compute node to use, time passes until the `POST /servers` call is issued. If we only passed the weight back from the `select` we would have to re-compute the appropriate compute node for the create command ... and we could end up with a different host. Somehow we need to remember the results of our computations and pass them outside of the Zone. Now, we could store this information in the local database and return a reference to it, but remember that the vast majority of weights are going be ignored. Storing them in the database would result in a flood of disk access and then we have to clean up all these entries periodically. Recall that there are going to be many many `select` calls issued to child Zones asking for estimates. +When `POST /zones/select` is called to estimate which compute node to use, time passes until the `POST /servers` call is issued. If we only passed the weight back from the `select` we would have to re-compute the appropriate compute node for the create command ... and we could end up with a different host. Somehow we need to remember the results of our computations and pass them outside of the Zone. Now, we could store this information in the local database and return a reference to it, but remember that the vast majority of weights are going to be ignored. Storing them in the database would result in a flood of disk access and then we have to clean up all these entries periodically. Recall that there are going to be many many `select` calls issued to child Zones asking for estimates. Instead, we take a rather innovative approach to the problem. We encrypt all the child zone internal details and pass them back the to parent Zone. If the parent zone decides to use a child Zone for the instance it simply passes the encrypted data back to the child during the `POST /servers` call as an extra parameter. The child Zone can then decrypt the hint and go directly to the Compute node previously selected. If the estimate isn't used, it is simply discarded by the parent. It's for this reason that it is so important that each Zone defines a unique encryption key via `--build_plan_encryption_key` @@ -108,7 +126,7 @@ NOTE: The features described in this section are related to the up-coming 'merge The OpenStack API allows a user to list all the instances they own via the `GET /servers/` command or the details on a particular instance via `GET /servers/###`. This mechanism is usually sufficient since OS API only allows for creating one instance at a time, unlike the EC2 API which allows you to specify a quantity of instances to be created. -NOTE: currently the `GET /servers` command is not Zone-aware since all operations done in child Zones are done via a single administrative account. Therefore, asking a child Zone to `GET /servers` would return all the active instances ... and that would be what the user intended. Later, when the Keystone Auth system is integrated with Nova, this functionality will be enabled. +NOTE: currently the `GET /servers` command is not Zone-aware since all operations done in child Zones are done via a single administrative account. Therefore, asking a child Zone to `GET /servers` would return all the active instances ... and that would not be what the user intended. Later, when the Keystone Auth system is integrated with Nova, this functionality will be enabled. We could use the OS API 1.1 Extensions mechanism to accept a `num_instances` parameter, but this would result in a different return code. Instead of getting back an `Instance` record, we would be getting back a `reservation_id`. So, instead, we've implemented a new command `POST /zones/boot` command which is nearly identical to `POST /servers` except that it takes a `num_instances` parameter and returns a `reservation_id`. Perhaps in OS API 2.x we can unify these approaches. @@ -117,7 +135,7 @@ Finally, we need to give the user a way to get information on each of the instan `python-novaclient` will be extended to support both of these changes. Host Filter --------------- +----------- As we mentioned earlier, filtering hosts is a very deployment-specific process. Service Providers may have a different set of criteria for filtering Compute nodes than a University. To faciliate this the `nova.scheduler.host_filter` module supports a variety of filtering strategies as well as an easy means for plugging in your own algorithms. @@ -130,21 +148,22 @@ The filter used is determined by the `--default_host_filter` flag, which points To create your own `HostFilter` the user simply has to derive from `nova.scheduler.host_filter.HostFilter` and implement two methods: `instance_type_to_filter` and `filter_hosts`. Since Nova is currently dependent on the `InstanceType` structure, the `instance_type_to_filter` method should take an `InstanceType` and turn it into an internal data structure usable by your filter. This is for backward compatibility with existing OpenStack and EC2 API calls. If you decide to create your own call for creating instances not based on `Flavors` or `InstanceTypes` you can ignore this method. The real work is done in `filter_hosts` which must return a list of host tuples for each appropriate host. The set of all available hosts is in the `ZoneManager` object passed into the call as well as the filter query. The host tuple contains (`<hostname>`, `<additional data>`) where `<additional data>` is whatever you want it to be. Cost Scheduler Weighing --------------- +----------------------- Every `ZoneAwareScheduler` derivation must also override the `weigh_hosts` method. This takes the list of filtered hosts (generated by the `filter_hosts` method) and returns a list of weight dicts. The weight dicts must contain two keys: `weight` and `hostname` where `weight` is simply an integer (lower is better) and `hostname` is the name of the host. The list does not need to be sorted, this will be done by the `ZoneAwareScheduler` base class when all the results have been assembled. Simple Zone Aware Scheduling --------------- -The easiest way to get started with the `ZoneAwareScheduler` is to use the `nova.scheduler.host_filter.HostFilterScheduler`. This scheduler uses the default Host Filter as and the `weight_hosts` method simply returns a weight of 1 for all hosts. But, from this, you can see calls being routed from Zone to Zone and follow the flow of things. +---------------------------- +The easiest way to get started with the `ZoneAwareScheduler` is to use the `nova.scheduler.host_filter.HostFilterScheduler`. This scheduler uses the default Host Filter and the `weight_hosts` method simply returns a weight of 1 for all hosts. But, from this, you can see calls being routed from Zone to Zone and follow the flow of things. The `--scheduler_driver` flag is how you specify the scheduler class name. Flags --------------- +----- All this Zone and Distributed Scheduler stuff can seem a little daunting to configure, but it's actually not too bad. Here are some of the main flags you should set in your `nova.conf` file: :: + --allow_admin_api=true --enable_zone_routing=true --zone_name=zone1 @@ -162,6 +181,7 @@ All this Zone and Distributed Scheduler stuff can seem a little daunting to conf Some optional flags which are handy for debugging are: :: + --connection_type=fake --verbose diff --git a/doc/source/devref/zone.rst b/doc/source/devref/zone.rst index 263560ee2..3dc0f80fd 100644 --- a/doc/source/devref/zone.rst +++ b/doc/source/devref/zone.rst @@ -21,7 +21,7 @@ A Nova deployment is called a Zone. A Zone allows you to partition your deployme The idea behind Zones is, if a particular deployment is not capable of servicing a particular request, the request may be forwarded to (child) Zones for possible processing. Zones may be nested in a tree fashion. -Zones only know about their immediate children, they do not know about their parent Zones and may in fact have more than one parent. Likewise, a Zone's children may themselves have child Zones. +Zones only know about their immediate children, they do not know about their parent Zones and may in fact have more than one parent. Likewise, a Zone's children may themselves have child Zones and, in those cases, the grandchild's internal structure would not be known to the grand-parent. Zones share nothing. They communicate via the public OpenStack API only. No database, queue, user or project definition is shared between Zones. @@ -99,7 +99,7 @@ You can get the `child zone api url`, `nova api key` and `username` from the `no export NOVA_URL="http://192.168.2.120:8774/v1.0/" -This equates to a POST operation to `.../zones/` to add a new zone. No connection attempt to the child zone is done when this command. It only puts an entry in the db at this point. After about 30 seconds the `ZoneManager` in the Scheduler services will attempt to talk to the child zone and get its information. +This equates to a POST operation to `.../zones/` to add a new zone. No connection attempt to the child zone is done with this command. It only puts an entry in the db at this point. After about 30 seconds the `ZoneManager` in the Scheduler services will attempt to talk to the child zone and get its information. Getting a list of child Zones ----------------------------- diff --git a/doc/source/image_src/zones_distsched_illustrations.odp b/doc/source/image_src/zones_distsched_illustrations.odp Binary files differnew file mode 100755 index 000000000..8762a183b --- /dev/null +++ b/doc/source/image_src/zones_distsched_illustrations.odp diff --git a/doc/source/images/costs_weights.png b/doc/source/images/costs_weights.png Binary files differnew file mode 100644 index 000000000..b65e98b0c --- /dev/null +++ b/doc/source/images/costs_weights.png diff --git a/doc/source/images/dating_service.png b/doc/source/images/dating_service.png Binary files differnew file mode 100644 index 000000000..49f1bd86a --- /dev/null +++ b/doc/source/images/dating_service.png diff --git a/doc/source/images/filtering.png b/doc/source/images/filtering.png Binary files differnew file mode 100644 index 000000000..4303bded8 --- /dev/null +++ b/doc/source/images/filtering.png diff --git a/doc/source/images/nova.compute.api.create.png b/doc/source/images/nova.compute.api.create.png Binary files differnew file mode 100755 index 000000000..999f39ed9 --- /dev/null +++ b/doc/source/images/nova.compute.api.create.png diff --git a/doc/source/images/nova.compute.api.create_all_at_once.png b/doc/source/images/nova.compute.api.create_all_at_once.png Binary files differnew file mode 100755 index 000000000..c3ce86d03 --- /dev/null +++ b/doc/source/images/nova.compute.api.create_all_at_once.png diff --git a/doc/source/images/zone_aware_overview.png b/doc/source/images/zone_aware_overview.png Binary files differnew file mode 100755 index 000000000..470e78138 --- /dev/null +++ b/doc/source/images/zone_aware_overview.png diff --git a/doc/source/images/zone_aware_scheduler.png b/doc/source/images/zone_aware_scheduler.png Binary files differnew file mode 100644 index 000000000..a144e1212 --- /dev/null +++ b/doc/source/images/zone_aware_scheduler.png diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py index 6672e60bb..7d78c5cfa 100644 --- a/nova/api/ec2/apirequest.py +++ b/nova/api/ec2/apirequest.py @@ -21,22 +21,15 @@ APIRequest class """ import datetime -import re # TODO(termie): replace minidom with etree from xml.dom import minidom from nova import log as logging +from nova.api.ec2 import ec2utils LOG = logging.getLogger("nova.api.request") -_c2u = re.compile('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))') - - -def _camelcase_to_underscore(str): - return _c2u.sub(r'_\1', str).lower().strip('_') - - def _underscore_to_camelcase(str): return ''.join([x[:1].upper() + x[1:] for x in str.split('_')]) @@ -51,59 +44,6 @@ def _database_to_isoformat(datetimeobj): return datetimeobj.strftime("%Y-%m-%dT%H:%M:%SZ") -def _try_convert(value): - """Return a non-string from a string or unicode, if possible. - - ============= ===================================================== - When value is returns - ============= ===================================================== - zero-length '' - 'None' None - 'True' True - 'False' False - '0', '-0' 0 - 0xN, -0xN int from hex (postitive) (N is any number) - 0bN, -0bN int from binary (positive) (N is any number) - * try conversion to int, float, complex, fallback value - - """ - if len(value) == 0: - return '' - if value == 'None': - return None - if value == 'True': - return True - if value == 'False': - return False - valueneg = value[1:] if value[0] == '-' else value - if valueneg == '0': - return 0 - if valueneg == '': - return value - if valueneg[0] == '0': - if valueneg[1] in 'xX': - return int(value, 16) - elif valueneg[1] in 'bB': - return int(value, 2) - else: - try: - return int(value, 8) - except ValueError: - pass - try: - return int(value) - except ValueError: - pass - try: - return float(value) - except ValueError: - pass - try: - return complex(value) - except ValueError: - return value - - class APIRequest(object): def __init__(self, controller, action, version, args): self.controller = controller @@ -114,7 +54,7 @@ class APIRequest(object): def invoke(self, context): try: method = getattr(self.controller, - _camelcase_to_underscore(self.action)) + ec2utils.camelcase_to_underscore(self.action)) except AttributeError: controller = self.controller action = self.action @@ -125,19 +65,7 @@ class APIRequest(object): # and reraise as 400 error. raise Exception(_error) - args = {} - for key, value in self.args.items(): - parts = key.split(".") - key = _camelcase_to_underscore(parts[0]) - if isinstance(value, str) or isinstance(value, unicode): - # NOTE(vish): Automatically convert strings back - # into their respective values - value = _try_convert(value) - if len(parts) > 1: - d = args.get(key, {}) - d[parts[1]] = value - value = d - args[key] = value + args = ec2utils.dict_from_dotted_str(self.args.items()) for key in args.keys(): # NOTE(vish): Turn numeric dict keys into lists diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 3fe463d69..9aaf37a2d 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -909,6 +909,25 @@ class CloudController(object): if kwargs.get('ramdisk_id'): ramdisk = self._get_image(context, kwargs['ramdisk_id']) kwargs['ramdisk_id'] = ramdisk['id'] + for bdm in kwargs.get('block_device_mapping', []): + # NOTE(yamahata) + # BlockDevicedMapping.<N>.DeviceName + # BlockDevicedMapping.<N>.Ebs.SnapshotId + # BlockDevicedMapping.<N>.Ebs.VolumeSize + # BlockDevicedMapping.<N>.Ebs.DeleteOnTermination + # BlockDevicedMapping.<N>.VirtualName + # => remove .Ebs and allow volume id in SnapshotId + ebs = bdm.pop('ebs', None) + if ebs: + ec2_id = ebs.pop('snapshot_id') + id = ec2utils.ec2_id_to_id(ec2_id) + if ec2_id.startswith('snap-'): + bdm['snapshot_id'] = id + elif ec2_id.startswith('vol-'): + bdm['volume_id'] = id + ebs.setdefault('delete_on_termination', True) + bdm.update(ebs) + image = self._get_image(context, kwargs['image_id']) if image: @@ -933,37 +952,54 @@ class CloudController(object): user_data=kwargs.get('user_data'), security_group=kwargs.get('security_group'), availability_zone=kwargs.get('placement', {}).get( - 'AvailabilityZone')) + 'AvailabilityZone'), + block_device_mapping=kwargs.get('block_device_mapping', {})) return self._format_run_instances(context, instances[0]['reservation_id']) + def _do_instance(self, action, context, ec2_id): + instance_id = ec2utils.ec2_id_to_id(ec2_id) + action(context, instance_id=instance_id) + + def _do_instances(self, action, context, instance_id): + for ec2_id in instance_id: + self._do_instance(action, context, ec2_id) + def terminate_instances(self, context, instance_id, **kwargs): """Terminate each instance in instance_id, which is a list of ec2 ids. instance_id is a kwarg so its name cannot be modified.""" LOG.debug(_("Going to start terminating instances")) - for ec2_id in instance_id: - instance_id = ec2utils.ec2_id_to_id(ec2_id) - self.compute_api.delete(context, instance_id=instance_id) + self._do_instances(self.compute_api.delete, context, instance_id) return True def reboot_instances(self, context, instance_id, **kwargs): """instance_id is a list of instance ids""" LOG.audit(_("Reboot instance %r"), instance_id, context=context) - for ec2_id in instance_id: - instance_id = ec2utils.ec2_id_to_id(ec2_id) - self.compute_api.reboot(context, instance_id=instance_id) + self._do_instances(self.compute_api.reboot, context, instance_id) + return True + + def stop_instances(self, context, instance_id, **kwargs): + """Stop each instances in instance_id. + Here instance_id is a list of instance ids""" + LOG.debug(_("Going to stop instances")) + self._do_instances(self.compute_api.stop, context, instance_id) + return True + + def start_instances(self, context, instance_id, **kwargs): + """Start each instances in instance_id. + Here instance_id is a list of instance ids""" + LOG.debug(_("Going to start instances")) + self._do_instances(self.compute_api.start, context, instance_id) return True def rescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" - instance_id = ec2utils.ec2_id_to_id(instance_id) - self.compute_api.rescue(context, instance_id=instance_id) + self._do_instance(self.compute_api.rescue, contect, instnace_id) return True def unrescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" - instance_id = ec2utils.ec2_id_to_id(instance_id) - self.compute_api.unrescue(context, instance_id=instance_id) + self._do_instance(self.compute_api.unrescue, context, instance_id) return True def update_instance(self, context, instance_id, **kwargs): @@ -974,7 +1010,8 @@ class CloudController(object): changes[field] = kwargs[field] if changes: instance_id = ec2utils.ec2_id_to_id(instance_id) - self.compute_api.update(context, instance_id=instance_id, **kwargs) + self.compute_api.update(context, instance_id=instance_id, + **changes) return True @staticmethod diff --git a/nova/api/ec2/ec2utils.py b/nova/api/ec2/ec2utils.py index 163aa4ed2..222e1de1e 100644 --- a/nova/api/ec2/ec2utils.py +++ b/nova/api/ec2/ec2utils.py @@ -16,6 +16,8 @@ # License for the specific language governing permissions and limitations # under the License. +import re + from nova import exception @@ -30,3 +32,95 @@ def ec2_id_to_id(ec2_id): def id_to_ec2_id(instance_id, template='i-%08x'): """Convert an instance ID (int) to an ec2 ID (i-[base 16 number])""" return template % instance_id + + +_c2u = re.compile('(((?<=[a-z])[A-Z])|([A-Z](?![A-Z]|$)))') + + +def camelcase_to_underscore(str): + return _c2u.sub(r'_\1', str).lower().strip('_') + + +def _try_convert(value): + """Return a non-string from a string or unicode, if possible. + + ============= ===================================================== + When value is returns + ============= ===================================================== + zero-length '' + 'None' None + 'True' True case insensitive + 'False' False case insensitive + '0', '-0' 0 + 0xN, -0xN int from hex (postitive) (N is any number) + 0bN, -0bN int from binary (positive) (N is any number) + * try conversion to int, float, complex, fallback value + + """ + if len(value) == 0: + return '' + if value == 'None': + return None + lowered_value = value.lower() + if lowered_value == 'true': + return True + if lowered_value == 'false': + return False + valueneg = value[1:] if value[0] == '-' else value + if valueneg == '0': + return 0 + if valueneg == '': + return value + if valueneg[0] == '0': + if valueneg[1] in 'xX': + return int(value, 16) + elif valueneg[1] in 'bB': + return int(value, 2) + else: + try: + return int(value, 8) + except ValueError: + pass + try: + return int(value) + except ValueError: + pass + try: + return float(value) + except ValueError: + pass + try: + return complex(value) + except ValueError: + return value + + +def dict_from_dotted_str(items): + """parse multi dot-separated argument into dict. + EBS boot uses multi dot-separeted arguments like + BlockDeviceMapping.1.DeviceName=snap-id + Convert the above into + {'block_device_mapping': {'1': {'device_name': snap-id}}} + """ + args = {} + for key, value in items: + parts = key.split(".") + key = camelcase_to_underscore(parts[0]) + if isinstance(value, str) or isinstance(value, unicode): + # NOTE(vish): Automatically convert strings back + # into their respective values + value = _try_convert(value) + + if len(parts) > 1: + d = args.get(key, {}) + args[key] = d + for k in parts[1:-1]: + k = camelcase_to_underscore(k) + v = d.get(k, {}) + d[k] = v + d = v + d[camelcase_to_underscore(parts[-1])] = value + else: + args[key] = value + + return args diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index c116e4220..f24017df0 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -81,7 +81,9 @@ class APIRouter(base_wsgi.Router): self._setup_routes(mapper) super(APIRouter, self).__init__(mapper) - def _setup_routes(self, mapper): + def _setup_routes(self, mapper, version): + """Routes common to all versions.""" + server_members = self.server_members server_members['action'] = 'POST' if FLAGS.allow_admin_api: @@ -98,11 +100,6 @@ class APIRouter(base_wsgi.Router): server_members['reset_network'] = 'POST' server_members['inject_network_info'] = 'POST' - mapper.resource("zone", "zones", - controller=zones.create_resource(), - collection={'detail': 'GET', 'info': 'GET', - 'select': 'POST'}) - mapper.resource("user", "users", controller=users.create_resource(), collection={'detail': 'GET'}) @@ -111,10 +108,33 @@ class APIRouter(base_wsgi.Router): controller=accounts.create_resource(), collection={'detail': 'GET'}) + mapper.resource("zone", "zones", + controller=zones.create_resource(version), + collection={'detail': 'GET', + 'info': 'GET', + 'select': 'POST', + 'boot': 'POST'}) + mapper.resource("console", "consoles", - controller=consoles.create_resource(), - parent_resource=dict(member_name='server', - collection_name='servers')) + controller=consoles.create_resource(), + parent_resource=dict(member_name='server', + collection_name='servers')) + + mapper.resource("server", "servers", + controller=servers.create_resource(version), + collection={'detail': 'GET'}, + member=self.server_members) + + mapper.resource("image", "images", + controller=images.create_resource(version), + collection={'detail': 'GET'}) + + mapper.resource("limit", "limits", + controller=limits.create_resource(version)) + + mapper.resource("flavor", "flavors", + controller=flavors.create_resource(version), + collection={'detail': 'GET'}) super(APIRouter, self).__init__(mapper) @@ -123,20 +143,11 @@ class APIRouterV10(APIRouter): """Define routes specific to OpenStack API V1.0.""" def _setup_routes(self, mapper): - super(APIRouterV10, self)._setup_routes(mapper) - mapper.resource("server", "servers", - controller=servers.create_resource('1.0'), - collection={'detail': 'GET'}, - member=self.server_members) - + super(APIRouterV10, self)._setup_routes(mapper, '1.0') mapper.resource("image", "images", controller=images.create_resource('1.0'), collection={'detail': 'GET'}) - mapper.resource("flavor", "flavors", - controller=flavors.create_resource('1.0'), - collection={'detail': 'GET'}) - mapper.resource("shared_ip_group", "shared_ip_groups", collection={'detail': 'GET'}, controller=shared_ip_groups.create_resource()) @@ -146,9 +157,6 @@ class APIRouterV10(APIRouter): parent_resource=dict(member_name='server', collection_name='servers')) - mapper.resource("limit", "limits", - controller=limits.create_resource('1.0')) - mapper.resource("ip", "ips", controller=ips.create_resource(), collection=dict(public='GET', private='GET'), parent_resource=dict(member_name='server', @@ -159,16 +167,7 @@ class APIRouterV11(APIRouter): """Define routes specific to OpenStack API V1.1.""" def _setup_routes(self, mapper): - super(APIRouterV11, self)._setup_routes(mapper) - mapper.resource("server", "servers", - controller=servers.create_resource('1.1'), - collection={'detail': 'GET'}, - member=self.server_members) - - mapper.resource("image", "images", - controller=images.create_resource('1.1'), - collection={'detail': 'GET'}) - + super(APIRouterV11, self)._setup_routes(mapper, '1.1') mapper.resource("image_meta", "meta", controller=image_metadata.create_resource(), parent_resource=dict(member_name='image', @@ -178,10 +177,3 @@ class APIRouterV11(APIRouter): controller=server_metadata.create_resource(), parent_resource=dict(member_name='server', collection_name='servers')) - - mapper.resource("flavor", "flavors", - controller=flavors.create_resource('1.1'), - collection={'detail': 'GET'}) - - mapper.resource("limit", "limits", - controller=limits.create_resource('1.1')) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index ce7e2805c..4da7ec0ef 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -26,8 +26,6 @@ from nova import log as logging LOG = logging.getLogger('nova.api.openstack.common') - - FLAGS = flags.FLAGS diff --git a/nova/api/openstack/contrib/volumes.py b/nova/api/openstack/contrib/volumes.py index feabdce89..e5e2c5b50 100644 --- a/nova/api/openstack/contrib/volumes.py +++ b/nova/api/openstack/contrib/volumes.py @@ -301,7 +301,7 @@ class Volumes(extensions.ExtensionDescriptor): return "Volumes" def get_alias(self): - return "VOLUMES" + return "os-volumes" def get_description(self): return "Volumes support" @@ -317,12 +317,12 @@ class Volumes(extensions.ExtensionDescriptor): # NOTE(justinsb): No way to provide singular name ('volume') # Does this matter? - res = extensions.ResourceExtension('volumes', + res = extensions.ResourceExtension('os-volumes', VolumeController(), collection_actions={'detail': 'GET'}) resources.append(res) - res = extensions.ResourceExtension('volume_attachments', + res = extensions.ResourceExtension('os-volume_attachments', VolumeAttachmentController(), parent=dict( member_name='server', diff --git a/nova/api/openstack/create_instance_helper.py b/nova/api/openstack/create_instance_helper.py new file mode 100644 index 000000000..436e524c1 --- /dev/null +++ b/nova/api/openstack/create_instance_helper.py @@ -0,0 +1,343 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import base64 +import re +import webob + +from webob import exc +from xml.dom import minidom + +from nova import exception +from nova import flags +from nova import log as logging +import nova.image +from nova import quota +from nova import utils + +from nova.compute import instance_types +from nova.api.openstack import faults +from nova.api.openstack import wsgi +from nova.auth import manager as auth_manager + + +LOG = logging.getLogger('nova.api.openstack.create_instance_helper') +FLAGS = flags.FLAGS + + +class CreateFault(exception.NovaException): + message = _("Invalid parameters given to create_instance.") + + def __init__(self, fault): + self.fault = fault + super(CreateFault, self).__init__() + + +class CreateInstanceHelper(object): + """This is the base class for OS API Controllers that + are capable of creating instances (currently Servers and Zones). + + Once we stabilize the Zones portion of the API we may be able + to move this code back into servers.py + """ + + def __init__(self, controller): + """We need the image service to create an instance.""" + self.controller = controller + self._image_service = utils.import_object(FLAGS.image_service) + super(CreateInstanceHelper, self).__init__() + + def create_instance(self, req, body, create_method): + """Creates a new server for the given user. The approach + used depends on the create_method. For example, the standard + POST /server call uses compute.api.create(), while + POST /zones/server uses compute.api.create_all_at_once(). + + The problem is, both approaches return different values (i.e. + [instance dicts] vs. reservation_id). So the handling of the + return type from this method is left to the caller. + """ + if not body: + raise faults.Fault(exc.HTTPUnprocessableEntity()) + + context = req.environ['nova.context'] + + password = self.controller._get_server_admin_password(body['server']) + + key_name = None + key_data = None + key_pairs = auth_manager.AuthManager.get_key_pairs(context) + if key_pairs: + key_pair = key_pairs[0] + key_name = key_pair['name'] + key_data = key_pair['public_key'] + + image_href = self.controller._image_ref_from_req_data(body) + try: + image_service, image_id = nova.image.get_image_service(image_href) + kernel_id, ramdisk_id = self._get_kernel_ramdisk_from_image( + req, image_id) + images = set([str(x['id']) for x in image_service.index(context)]) + assert str(image_id) in images + except Exception, e: + msg = _("Cannot find requested image %(image_href)s: %(e)s" % + locals()) + raise faults.Fault(exc.HTTPBadRequest(explanation=msg)) + + personality = body['server'].get('personality') + + injected_files = [] + if personality: + injected_files = self._get_injected_files(personality) + + flavor_id = self.controller._flavor_id_from_req_data(body) + + if not 'name' in body['server']: + msg = _("Server name is not defined") + raise exc.HTTPBadRequest(explanation=msg) + + zone_blob = body['server'].get('blob') + name = body['server']['name'] + self._validate_server_name(name) + name = name.strip() + + reservation_id = body['server'].get('reservation_id') + + try: + inst_type = \ + instance_types.get_instance_type_by_flavor_id(flavor_id) + extra_values = { + 'instance_type': inst_type, + 'image_ref': image_href, + 'password': password} + + return (extra_values, + create_method(context, + inst_type, + image_id, + kernel_id=kernel_id, + ramdisk_id=ramdisk_id, + display_name=name, + display_description=name, + key_name=key_name, + key_data=key_data, + metadata=body['server'].get('metadata', {}), + injected_files=injected_files, + admin_password=password, + zone_blob=zone_blob, + reservation_id=reservation_id)) + except quota.QuotaError as error: + self._handle_quota_error(error) + except exception.ImageNotFound as error: + msg = _("Can not find requested image") + raise faults.Fault(exc.HTTPBadRequest(explanation=msg)) + + # Let the caller deal with unhandled exceptions. + + def _handle_quota_error(self, error): + """ + Reraise quota errors as api-specific http exceptions + """ + if error.code == "OnsetFileLimitExceeded": + expl = _("Personality file limit exceeded") + raise exc.HTTPBadRequest(explanation=expl) + if error.code == "OnsetFilePathLimitExceeded": + expl = _("Personality file path too long") + raise exc.HTTPBadRequest(explanation=expl) + if error.code == "OnsetFileContentLimitExceeded": + expl = _("Personality file content too long") + raise exc.HTTPBadRequest(explanation=expl) + # if the original error is okay, just reraise it + raise error + + def _deserialize_create(self, request): + """ + Deserialize a create request + + Overrides normal behavior in the case of xml content + """ + if request.content_type == "application/xml": + deserializer = ServerCreateRequestXMLDeserializer() + return deserializer.deserialize(request.body) + else: + return self._deserialize(request.body, request.get_content_type()) + + def _validate_server_name(self, value): + if not isinstance(value, basestring): + msg = _("Server name is not a string or unicode") + raise exc.HTTPBadRequest(explanation=msg) + + if value.strip() == '': + msg = _("Server name is an empty string") + raise exc.HTTPBadRequest(explanation=msg) + + def _get_kernel_ramdisk_from_image(self, req, image_id): + """Fetch an image from the ImageService, then if present, return the + associated kernel and ramdisk image IDs. + """ + context = req.environ['nova.context'] + image_meta = self._image_service.show(context, image_id) + # NOTE(sirp): extracted to a separate method to aid unit-testing, the + # new method doesn't need a request obj or an ImageService stub + kernel_id, ramdisk_id = self._do_get_kernel_ramdisk_from_image( + image_meta) + return kernel_id, ramdisk_id + + @staticmethod + def _do_get_kernel_ramdisk_from_image(image_meta): + """Given an ImageService image_meta, return kernel and ramdisk image + ids if present. + + This is only valid for `ami` style images. + """ + image_id = image_meta['id'] + if image_meta['status'] != 'active': + raise exception.ImageUnacceptable(image_id=image_id, + reason=_("status is not active")) + + if image_meta.get('container_format') != 'ami': + return None, None + + try: + kernel_id = image_meta['properties']['kernel_id'] + except KeyError: + raise exception.KernelNotFoundForImage(image_id=image_id) + + try: + ramdisk_id = image_meta['properties']['ramdisk_id'] + except KeyError: + raise exception.RamdiskNotFoundForImage(image_id=image_id) + + return kernel_id, ramdisk_id + + def _get_injected_files(self, personality): + """ + Create a list of injected files from the personality attribute + + At this time, injected_files must be formatted as a list of + (file_path, file_content) pairs for compatibility with the + underlying compute service. + """ + injected_files = [] + + for item in personality: + try: + path = item['path'] + contents = item['contents'] + except KeyError as key: + expl = _('Bad personality format: missing %s') % key + raise exc.HTTPBadRequest(explanation=expl) + except TypeError: + expl = _('Bad personality format') + raise exc.HTTPBadRequest(explanation=expl) + try: + contents = base64.b64decode(contents) + except TypeError: + expl = _('Personality content for %s cannot be decoded') % path + raise exc.HTTPBadRequest(explanation=expl) + injected_files.append((path, contents)) + return injected_files + + def _get_server_admin_password_old_style(self, server): + """ Determine the admin password for a server on creation """ + return utils.generate_password(16) + + def _get_server_admin_password_new_style(self, server): + """ Determine the admin password for a server on creation """ + password = server.get('adminPass') + + if password is None: + return utils.generate_password(16) + if not isinstance(password, basestring) or password == '': + msg = _("Invalid adminPass") + raise exc.HTTPBadRequest(explanation=msg) + return password + + +class ServerXMLDeserializer(wsgi.XMLDeserializer): + """ + Deserializer to handle xml-formatted server create requests. + + Handles standard server attributes as well as optional metadata + and personality attributes + """ + + def create(self, string): + """Deserialize an xml-formatted server create request""" + dom = minidom.parseString(string) + server = self._extract_server(dom) + return {'server': server} + + def _extract_server(self, node): + """Marshal the server attribute of a parsed request""" + server = {} + server_node = self._find_first_child_named(node, 'server') + for attr in ["name", "imageId", "flavorId", "imageRef", "flavorRef"]: + if server_node.getAttribute(attr): + server[attr] = server_node.getAttribute(attr) + metadata = self._extract_metadata(server_node) + if metadata is not None: + server["metadata"] = metadata + personality = self._extract_personality(server_node) + if personality is not None: + server["personality"] = personality + return server + + def _extract_metadata(self, server_node): + """Marshal the metadata attribute of a parsed request""" + metadata_node = self._find_first_child_named(server_node, "metadata") + if metadata_node is None: + return None + metadata = {} + for meta_node in self._find_children_named(metadata_node, "meta"): + key = meta_node.getAttribute("key") + metadata[key] = self._extract_text(meta_node) + return metadata + + def _extract_personality(self, server_node): + """Marshal the personality attribute of a parsed request""" + personality_node = \ + self._find_first_child_named(server_node, "personality") + if personality_node is None: + return None + personality = [] + for file_node in self._find_children_named(personality_node, "file"): + item = {} + if file_node.hasAttribute("path"): + item["path"] = file_node.getAttribute("path") + item["contents"] = self._extract_text(file_node) + personality.append(item) + return personality + + def _find_first_child_named(self, parent, name): + """Search a nodes children for the first child with a given name""" + for node in parent.childNodes: + if node.nodeName == name: + return node + return None + + def _find_children_named(self, parent, name): + """Return all of a nodes children who have the given name""" + for node in parent.childNodes: + if node.nodeName == name: + yield node + + def _extract_text(self, node): + """Get the text field contained by the given node""" + if len(node.childNodes) == 1: + child = node.childNodes[0] + if child.nodeType == child.TEXT_NODE: + return child.nodeValue + return "" diff --git a/nova/api/openstack/extensions.py b/nova/api/openstack/extensions.py index 54e17e23d..da06ecd15 100644 --- a/nova/api/openstack/extensions.py +++ b/nova/api/openstack/extensions.py @@ -374,6 +374,8 @@ class ExtensionManager(object): LOG.debug(_('Ext updated: %s'), extension.get_updated()) except AttributeError as ex: LOG.exception(_("Exception loading extension: %s"), unicode(ex)) + return False + return True def _load_all_extensions(self): """Load extensions from the configured path. @@ -412,15 +414,16 @@ class ExtensionManager(object): 'file': ext_path}) continue new_ext = new_ext_class() - self._check_extension(new_ext) - self._add_extension(new_ext) + self.add_extension(new_ext) + + def add_extension(self, ext): + # Do nothing if the extension doesn't check out + if not self._check_extension(ext): + return - def _add_extension(self, ext): alias = ext.get_alias() LOG.audit(_('Loaded extension: %s'), alias) - self._check_extension(ext) - if alias in self.extensions: raise exception.Error("Found duplicate extension: %s" % alias) self.extensions[alias] = ext diff --git a/nova/api/openstack/ips.py b/nova/api/openstack/ips.py index abea71830..71646b6d3 100644 --- a/nova/api/openstack/ips.py +++ b/nova/api/openstack/ips.py @@ -32,25 +32,24 @@ class Controller(object): self.compute_api = nova.compute.API() self.builder = nova.api.openstack.views.addresses.ViewBuilderV10() - def index(self, req, server_id): + def _get_instance(self, req, server_id): try: - instance = self.compute_api.get(req.environ['nova.context'], id) + instance = self.compute_api.get( + req.environ['nova.context'], server_id) except nova.exception.NotFound: return faults.Fault(exc.HTTPNotFound()) + return instance + + def index(self, req, server_id): + instance = self._get_instance(req, server_id) return {'addresses': self.builder.build(instance)} def public(self, req, server_id): - try: - instance = self.compute_api.get(req.environ['nova.context'], id) - except nova.exception.NotFound: - return faults.Fault(exc.HTTPNotFound()) + instance = self._get_instance(req, server_id) return {'public': self.builder.build_public_parts(instance)} def private(self, req, server_id): - try: - instance = self.compute_api.get(req.environ['nova.context'], id) - except nova.exception.NotFound: - return faults.Fault(exc.HTTPNotFound()) + instance = self._get_instance(req, server_id) return {'private': self.builder.build_private_parts(instance)} def show(self, req, server_id, id): diff --git a/nova/api/openstack/limits.py b/nova/api/openstack/limits.py index dc2bc6bbc..fede96e33 100644 --- a/nova/api/openstack/limits.py +++ b/nova/api/openstack/limits.py @@ -99,7 +99,7 @@ def create_resource(version='1.0'): serializers = { 'application/xml': wsgi.XMLDictSerializer(xmlns=xmlns, - metadata=metadata) + metadata=metadata), } return wsgi.Resource(controller, serializers=serializers) diff --git a/nova/api/openstack/notes.txt b/nova/api/openstack/notes.txt index 2330f1002..4e95bffc8 100644 --- a/nova/api/openstack/notes.txt +++ b/nova/api/openstack/notes.txt @@ -7,9 +7,6 @@ image ids. GlanceImageService(ImageService): image ids are URIs. -LocalImageService(ImageService): -image ids are random strings. - OpenstackAPITranslationStore: translates RS server/images/flavor/etc ids into formats required by a given ImageService strategy. diff --git a/nova/api/openstack/server_metadata.py b/nova/api/openstack/server_metadata.py index 57666f6b7..8a314de22 100644 --- a/nova/api/openstack/server_metadata.py +++ b/nova/api/openstack/server_metadata.py @@ -18,9 +18,10 @@ from webob import exc from nova import compute -from nova import quota from nova.api.openstack import faults from nova.api.openstack import wsgi +from nova import exception +from nova import quota class Controller(object): @@ -45,7 +46,11 @@ class Controller(object): def index(self, req, server_id): """ Returns the list of metadata for a given instance """ context = req.environ['nova.context'] - return self._get_metadata(context, server_id) + try: + return self._get_metadata(context, server_id) + except exception.InstanceNotFound: + msg = _('Server %(server_id)s does not exist') % locals() + raise exc.HTTPNotFound(explanation=msg) def create(self, req, server_id, body): self._check_body(body) @@ -55,8 +60,13 @@ class Controller(object): self.compute_api.update_or_create_instance_metadata(context, server_id, metadata) + except exception.InstanceNotFound: + msg = _('Server %(server_id)s does not exist') % locals() + raise exc.HTTPNotFound(explanation=msg) + except quota.QuotaError as error: self._handle_quota_error(error) + return body def update(self, req, server_id, id, body): @@ -72,6 +82,10 @@ class Controller(object): self.compute_api.update_or_create_instance_metadata(context, server_id, body) + except exception.InstanceNotFound: + msg = _('Server %(server_id)s does not exist') % locals() + raise exc.HTTPNotFound(explanation=msg) + except quota.QuotaError as error: self._handle_quota_error(error) @@ -80,16 +94,26 @@ class Controller(object): def show(self, req, server_id, id): """ Return a single metadata item """ context = req.environ['nova.context'] - data = self._get_metadata(context, server_id) - if id in data['metadata']: + try: + data = self._get_metadata(context, server_id) + except exception.InstanceNotFound: + msg = _('Server %(server_id)s does not exist') % locals() + raise exc.HTTPNotFound(explanation=msg) + + try: return {id: data['metadata'][id]} - else: - return faults.Fault(exc.HTTPNotFound()) + except KeyError: + msg = _("metadata item %s was not found" % (id)) + raise exc.HTTPNotFound(explanation=msg) def delete(self, req, server_id, id): """ Deletes an existing metadata """ context = req.environ['nova.context'] - self.compute_api.delete_instance_metadata(context, server_id, id) + try: + self.compute_api.delete_instance_metadata(context, server_id, id) + except exception.InstanceNotFound: + msg = _('Server %(server_id)s does not exist') % locals() + raise exc.HTTPNotFound(explanation=msg) def _handle_quota_error(self, error): """Reraise quota errors as api-specific http exceptions.""" diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 9cf5e8721..b82a6de19 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -17,24 +17,20 @@ import base64 import traceback from webob import exc -from xml.dom import minidom from nova import compute from nova import exception from nova import flags -import nova.image from nova import log as logging -from nova import quota from nova import utils from nova.api.openstack import common +from nova.api.openstack import create_instance_helper as helper from nova.api.openstack import faults import nova.api.openstack.views.addresses import nova.api.openstack.views.flavors import nova.api.openstack.views.images import nova.api.openstack.views.servers from nova.api.openstack import wsgi -from nova.auth import manager as auth_manager -from nova.compute import instance_types import nova.api.openstack from nova.scheduler import api as scheduler_api @@ -48,14 +44,14 @@ class Controller(object): def __init__(self): self.compute_api = compute.API() - self._image_service = utils.import_object(FLAGS.image_service) + self.helper = helper.CreateInstanceHelper(self) def index(self, req): """ Returns a list of server names and ids for a given user """ try: servers = self._items(req, is_detail=False) except exception.Invalid as err: - return exc.HTTPBadRequest(str(err)) + return exc.HTTPBadRequest(explanation=str(err)) return servers def detail(self, req): @@ -63,15 +59,9 @@ class Controller(object): try: servers = self._items(req, is_detail=True) except exception.Invalid as err: - return exc.HTTPBadRequest(str(err)) + return exc.HTTPBadRequest(explanation=str(err)) return servers - def _image_ref_from_req_data(self, data): - raise NotImplementedError() - - def _flavor_id_from_req_data(self, data): - raise NotImplementedError() - def _get_view_builder(self, req): raise NotImplementedError() @@ -86,7 +76,10 @@ class Controller(object): builder - the response model builder """ - instance_list = self.compute_api.get_all(req.environ['nova.context']) + reservation_id = req.str_GET.get('reservation_id') + instance_list = self.compute_api.get_all( + req.environ['nova.context'], + reservation_id=reservation_id) limited_list = self._limit_items(instance_list, req) builder = self._get_view_builder(req) servers = [builder.build(inst, is_detail)['server'] @@ -115,128 +108,25 @@ class Controller(object): def create(self, req, body): """ Creates a new server for a given user """ - if not body: - return faults.Fault(exc.HTTPUnprocessableEntity()) - - context = req.environ['nova.context'] - - password = self._get_server_admin_password(body['server']) - - key_name = None - key_data = None - key_pairs = auth_manager.AuthManager.get_key_pairs(context) - if key_pairs: - key_pair = key_pairs[0] - key_name = key_pair['name'] - key_data = key_pair['public_key'] - - image_href = self._image_ref_from_req_data(body) + extra_values = None + result = None try: - image_service, image_id = nova.image.get_image_service(image_href) - kernel_id, ramdisk_id = self._get_kernel_ramdisk_from_image( - req, image_service, image_id) - images = set([str(x['id']) for x in image_service.index(context)]) - assert str(image_id) in images - except: - msg = _("Cannot find requested image %s") % image_href - return faults.Fault(exc.HTTPBadRequest(msg)) - - personality = body['server'].get('personality') + extra_values, result = self.helper.create_instance( + req, body, self.compute_api.create) + except faults.Fault, f: + return f - injected_files = [] - if personality: - injected_files = self._get_injected_files(personality) + instances = result - flavor_id = self._flavor_id_from_req_data(body) - - if not 'name' in body['server']: - msg = _("Server name is not defined") - return exc.HTTPBadRequest(msg) - - zone_blob = body['server'].get('blob') - name = body['server']['name'] - self._validate_server_name(name) - name = name.strip() - - try: - inst_type = \ - instance_types.get_instance_type_by_flavor_id(flavor_id) - (inst,) = self.compute_api.create( - context, - inst_type, - image_href, - kernel_id=kernel_id, - ramdisk_id=ramdisk_id, - display_name=name, - display_description=name, - key_name=key_name, - key_data=key_data, - metadata=body['server'].get('metadata', {}), - injected_files=injected_files, - admin_password=password, - zone_blob=zone_blob) - except quota.QuotaError as error: - self._handle_quota_error(error) - except exception.ImageNotFound as error: - msg = _("Can not find requested image") - return faults.Fault(exc.HTTPBadRequest(msg)) - - inst['instance_type'] = inst_type - inst['image_ref'] = image_href + (inst, ) = instances + for key in ['instance_type', 'image_ref']: + inst[key] = extra_values[key] builder = self._get_view_builder(req) server = builder.build(inst, is_detail=True) - server['server']['adminPass'] = password + server['server']['adminPass'] = extra_values['password'] return server - def _get_injected_files(self, personality): - """ - Create a list of injected files from the personality attribute - - At this time, injected_files must be formatted as a list of - (file_path, file_content) pairs for compatibility with the - underlying compute service. - """ - injected_files = [] - - for item in personality: - try: - path = item['path'] - contents = item['contents'] - except KeyError as key: - expl = _('Bad personality format: missing %s') % key - raise exc.HTTPBadRequest(explanation=expl) - except TypeError: - expl = _('Bad personality format') - raise exc.HTTPBadRequest(explanation=expl) - try: - contents = base64.b64decode(contents) - except TypeError: - expl = _('Personality content for %s cannot be decoded') % path - raise exc.HTTPBadRequest(explanation=expl) - injected_files.append((path, contents)) - return injected_files - - def _handle_quota_error(self, error): - """ - Reraise quota errors as api-specific http exceptions - """ - if error.code == "OnsetFileLimitExceeded": - expl = _("Personality file limit exceeded") - raise exc.HTTPBadRequest(explanation=expl) - if error.code == "OnsetFilePathLimitExceeded": - expl = _("Personality file path too long") - raise exc.HTTPBadRequest(explanation=expl) - if error.code == "OnsetFileContentLimitExceeded": - expl = _("Personality file content too long") - raise exc.HTTPBadRequest(explanation=expl) - # if the original error is okay, just reraise it - raise error - - def _get_server_admin_password(self, server): - """ Determine the admin password for a server on creation """ - return utils.generate_password(16) - @scheduler_api.redirect_handler def update(self, req, id, body): """ Updates the server name or password """ @@ -251,7 +141,7 @@ class Controller(object): if 'name' in body['server']: name = body['server']['name'] - self._validate_server_name(name) + self.helper._validate_server_name(name) update_dict['display_name'] = name.strip() self._parse_update(ctxt, id, body, update_dict) @@ -263,15 +153,6 @@ class Controller(object): return exc.HTTPNoContent() - def _validate_server_name(self, value): - if not isinstance(value, basestring): - msg = _("Server name is not a string or unicode") - raise exc.HTTPBadRequest(msg) - - if value.strip() == '': - msg = _("Server name is an empty string") - raise exc.HTTPBadRequest(msg) - def _parse_update(self, context, id, inst_dict, update_dict): pass @@ -520,45 +401,9 @@ class Controller(object): error=item.error)) return dict(actions=actions) - def _get_kernel_ramdisk_from_image(self, req, image_service, image_id): - """Fetch an image from the ImageService, then if present, return the - associated kernel and ramdisk image IDs. - """ - context = req.environ['nova.context'] - image_meta = image_service.show(context, image_id) - # NOTE(sirp): extracted to a separate method to aid unit-testing, the - # new method doesn't need a request obj or an ImageService stub - return self._do_get_kernel_ramdisk_from_image(image_meta) - - @staticmethod - def _do_get_kernel_ramdisk_from_image(image_meta): - """Given an ImageService image_meta, return kernel and ramdisk image - ids if present. - - This is only valid for `ami` style images. - """ - image_id = image_meta['id'] - if image_meta['status'] != 'active': - raise exception.ImageUnacceptable(image_id=image_id, - reason=_("status is not active")) - - if image_meta.get('container_format') != 'ami': - return None, None - - try: - kernel_id = image_meta['properties']['kernel_id'] - except KeyError: - raise exception.KernelNotFoundForImage(image_id=image_id) - - try: - ramdisk_id = image_meta['properties']['ramdisk_id'] - except KeyError: - raise exception.RamdiskNotFoundForImage(image_id=image_id) - - return kernel_id, ramdisk_id - class ControllerV10(Controller): + def _image_ref_from_req_data(self, data): return data['server']['imageId'] @@ -615,6 +460,10 @@ class ControllerV10(Controller): response.empty_body = True return response + def _get_server_admin_password(self, server): + """ Determine the admin password for a server on creation """ + return self.helper._get_server_admin_password_old_style(server) + class ControllerV11(Controller): def _image_ref_from_req_data(self, data): @@ -639,11 +488,11 @@ class ControllerV11(Controller): if (not 'changePassword' in input_dict or not 'adminPass' in input_dict['changePassword']): msg = _("No adminPass was specified") - return exc.HTTPBadRequest(msg) + return exc.HTTPBadRequest(explanation=msg) password = input_dict['changePassword']['adminPass'] if not isinstance(password, basestring) or password == '': msg = _("Invalid adminPass") - return exc.HTTPBadRequest(msg) + return exc.HTTPBadRequest(explanation=msg) self.compute_api.set_admin_password(context, id, password) return exc.HTTPAccepted() @@ -724,92 +573,12 @@ class ControllerV11(Controller): response.empty_body = True return response + def get_default_xmlns(self, req): + return common.XML_NS_V11 + def _get_server_admin_password(self, server): """ Determine the admin password for a server on creation """ - password = server.get('adminPass') - if password is None: - return utils.generate_password(16) - if not isinstance(password, basestring) or password == '': - msg = _("Invalid adminPass") - raise exc.HTTPBadRequest(msg) - return password - - -class ServerXMLDeserializer(wsgi.XMLDeserializer): - """ - Deserializer to handle xml-formatted server create requests. - - Handles standard server attributes as well as optional metadata - and personality attributes - """ - - def create(self, string): - """Deserialize an xml-formatted server create request""" - dom = minidom.parseString(string) - server = self._extract_server(dom) - return {'server': server} - - def _extract_server(self, node): - """Marshal the server attribute of a parsed request""" - server = {} - server_node = self._find_first_child_named(node, 'server') - for attr in ["name", "imageId", "flavorId", "imageRef", "flavorRef"]: - if server_node.getAttribute(attr): - server[attr] = server_node.getAttribute(attr) - metadata = self._extract_metadata(server_node) - if metadata is not None: - server["metadata"] = metadata - personality = self._extract_personality(server_node) - if personality is not None: - server["personality"] = personality - return server - - def _extract_metadata(self, server_node): - """Marshal the metadata attribute of a parsed request""" - metadata_node = self._find_first_child_named(server_node, "metadata") - if metadata_node is None: - return None - metadata = {} - for meta_node in self._find_children_named(metadata_node, "meta"): - key = meta_node.getAttribute("key") - metadata[key] = self._extract_text(meta_node) - return metadata - - def _extract_personality(self, server_node): - """Marshal the personality attribute of a parsed request""" - personality_node = \ - self._find_first_child_named(server_node, "personality") - if personality_node is None: - return None - personality = [] - for file_node in self._find_children_named(personality_node, "file"): - item = {} - if file_node.hasAttribute("path"): - item["path"] = file_node.getAttribute("path") - item["contents"] = self._extract_text(file_node) - personality.append(item) - return personality - - def _find_first_child_named(self, parent, name): - """Search a nodes children for the first child with a given name""" - for node in parent.childNodes: - if node.nodeName == name: - return node - return None - - def _find_children_named(self, parent, name): - """Return all of a nodes children who have the given name""" - for node in parent.childNodes: - if node.nodeName == name: - yield node - - def _extract_text(self, node): - """Get the text field contained by the given node""" - if len(node.childNodes) == 1: - child = node.childNodes[0] - if child.nodeType == child.TEXT_NODE: - return child.nodeValue - return "" + return self.helper._get_server_admin_password_new_style(server) def create_resource(version='1.0'): @@ -845,7 +614,7 @@ def create_resource(version='1.0'): } deserializers = { - 'application/xml': ServerXMLDeserializer(), + 'application/xml': helper.ServerXMLDeserializer(), } return wsgi.Resource(controller, serializers=serializers, diff --git a/nova/api/openstack/views/servers.py b/nova/api/openstack/views/servers.py index b2352e3fd..cbfa5aae7 100644 --- a/nova/api/openstack/views/servers.py +++ b/nova/api/openstack/views/servers.py @@ -42,12 +42,15 @@ class ViewBuilder(object): def build(self, inst, is_detail): """Return a dict that represenst a server.""" - if is_detail: - server = self._build_detail(inst) + if inst.get('_is_precooked', False): + server = dict(server=inst) else: - server = self._build_simple(inst) + if is_detail: + server = self._build_detail(inst) + else: + server = self._build_simple(inst) - self._build_extra(server, inst) + self._build_extra(server, inst) return server @@ -72,13 +75,14 @@ class ViewBuilder(object): } inst_dict = { - 'id': int(inst['id']), + 'id': inst['id'], 'name': inst['display_name'], 'addresses': self.addresses_builder.build(inst), 'status': power_mapping[inst.get('state')]} ctxt = nova.context.get_admin_context() compute_api = nova.compute.API() + if compute_api.has_finished_migration(ctxt, inst['id']): inst_dict['status'] = 'RESIZE-CONFIRM' @@ -95,6 +99,7 @@ class ViewBuilder(object): self._build_image(inst_dict, inst) self._build_flavor(inst_dict, inst) + inst_dict['uuid'] = inst['uuid'] return dict(server=inst_dict) def _build_image(self, response, inst): diff --git a/nova/api/openstack/wsgi.py b/nova/api/openstack/wsgi.py index 6760735c4..a57b7f72b 100644 --- a/nova/api/openstack/wsgi.py +++ b/nova/api/openstack/wsgi.py @@ -2,7 +2,9 @@ import json import webob from xml.dom import minidom +from xml.parsers import expat +import faults from nova import exception from nova import log as logging from nova import utils @@ -60,7 +62,7 @@ class TextDeserializer(object): def deserialize(self, datastring, action='default'): """Find local deserialization method and parse request body.""" - action_method = getattr(self, action, self.default) + action_method = getattr(self, str(action), self.default) return action_method(datastring) def default(self, datastring): @@ -71,7 +73,11 @@ class TextDeserializer(object): class JSONDeserializer(TextDeserializer): def default(self, datastring): - return utils.loads(datastring) + try: + return utils.loads(datastring) + except ValueError: + raise exception.MalformedRequestBody( + reason=_("malformed JSON in request body")) class XMLDeserializer(TextDeserializer): @@ -86,8 +92,13 @@ class XMLDeserializer(TextDeserializer): def default(self, datastring): plurals = set(self.metadata.get('plurals', {})) - node = minidom.parseString(datastring).childNodes[0] - return {node.nodeName: self._from_xml_node(node, plurals)} + + try: + node = minidom.parseString(datastring).childNodes[0] + return {node.nodeName: self._from_xml_node(node, plurals)} + except expat.ExpatError: + raise exception.MalformedRequestBody( + reason=_("malformed XML in request body")) def _from_xml_node(self, node, listnames): """Convert a minidom node to a simple Python type. @@ -189,7 +200,7 @@ class DictSerializer(object): def serialize(self, data, action='default'): """Find local serialization method and encode response body.""" - action_method = getattr(self, action, self.default) + action_method = getattr(self, str(action), self.default) return action_method(data) def default(self, data): @@ -296,7 +307,7 @@ class ResponseSerializer(object): } self.serializers.update(serializers or {}) - def serialize(self, response_data, content_type): + def serialize(self, response_data, content_type, action='default'): """Serialize a dict into a string and wrap in a wsgi.Request object. :param response_data: dict produced by the Controller @@ -307,7 +318,7 @@ class ResponseSerializer(object): response.headers['Content-Type'] = content_type serializer = self.get_serializer(content_type) - response.body = serializer.serialize(response_data) + response.body = serializer.serialize(response_data, action) return response @@ -352,22 +363,26 @@ class Resource(wsgi.Application): action, action_args, accept = self.deserializer.deserialize( request) except exception.InvalidContentType: - return webob.exc.HTTPBadRequest(_("Unsupported Content-Type")) + msg = _("Unsupported Content-Type") + return webob.exc.HTTPBadRequest(explanation=msg) + except exception.MalformedRequestBody: + msg = _("Malformed request body") + return faults.Fault(webob.exc.HTTPBadRequest(explanation=msg)) action_result = self.dispatch(request, action, action_args) #TODO(bcwaldon): find a more elegant way to pass through non-dict types if type(action_result) is dict: - response = self.serializer.serialize(action_result, accept) + response = self.serializer.serialize(action_result, accept, action) else: response = action_result try: msg_dict = dict(url=request.url, status=response.status_int) msg = _("%(url)s returned with HTTP %(status)d") % msg_dict - except AttributeError: - msg_dict = dict(url=request.url) - msg = _("%(url)s returned a fault") + except AttributeError, e: + msg_dict = dict(url=request.url, e=e) + msg = _("%(url)s returned a fault: %(e)s" % msg_dict) LOG.debug(msg) diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index b2f7898cb..8864f825b 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -21,9 +21,14 @@ from nova import db from nova import exception from nova import flags from nova import log as logging + +from nova.compute import api as compute +from nova.scheduler import api + +from nova.api.openstack import create_instance_helper as helper from nova.api.openstack import common +from nova.api.openstack import faults from nova.api.openstack import wsgi -from nova.scheduler import api FLAGS = flags.FLAGS @@ -59,6 +64,11 @@ def check_encryption_key(func): class Controller(object): + """Controller for Zone resources.""" + + def __init__(self): + self.compute_api = compute.API() + self.helper = helper.CreateInstanceHelper(self) def index(self, req): """Return all zones in brief""" @@ -93,21 +103,39 @@ class Controller(object): return dict(zone=_scrub_zone(zone)) def delete(self, req, id): + """Delete a child zone entry.""" zone_id = int(id) api.zone_delete(req.environ['nova.context'], zone_id) return {} def create(self, req, body): + """Create a child zone entry.""" context = req.environ['nova.context'] zone = api.zone_create(context, body["zone"]) return dict(zone=_scrub_zone(zone)) def update(self, req, id, body): + """Update a child zone entry.""" context = req.environ['nova.context'] zone_id = int(id) zone = api.zone_update(context, zone_id, body["zone"]) return dict(zone=_scrub_zone(zone)) + def boot(self, req, body): + """Creates a new server for a given user while being Zone aware. + + Returns a reservation ID (a UUID). + """ + result = None + try: + extra_values, result = self.helper.create_instance(req, body, + self.compute_api.create_all_at_once) + except faults.Fault, f: + return f + + reservation_id = result + return {'reservation_id': reservation_id} + @check_encryption_key def select(self, req, body): """Returns a weighted list of costs to create instances @@ -131,8 +159,37 @@ class Controller(object): blob=cipher_text)) return cooked + def _image_ref_from_req_data(self, data): + return data['server']['imageId'] + + def _flavor_id_from_req_data(self, data): + return data['server']['flavorId'] + + def _get_server_admin_password(self, server): + """ Determine the admin password for a server on creation """ + return self.helper._get_server_admin_password_old_style(server) + + +class ControllerV11(object): + """Controller for 1.1 Zone resources.""" + + def _get_server_admin_password(self, server): + """ Determine the admin password for a server on creation """ + return self.helper._get_server_admin_password_new_style(server) + + def _image_ref_from_req_data(self, data): + return data['server']['imageRef'] + + def _flavor_id_from_req_data(self, data): + return data['server']['flavorRef'] + + +def create_resource(version): + controller = { + '1.0': Controller, + '1.1': ControllerV11, + }[version]() -def create_resource(): metadata = { "attributes": { "zone": ["id", "api_url", "name", "capabilities"], @@ -144,4 +201,9 @@ def create_resource(): metadata=metadata), } - return wsgi.Resource(Controller(), serializers=serializers) + deserializers = { + 'application/xml': helper.ServerXMLDeserializer(), + } + + return wsgi.Resource(controller, serializers=serializers, + deserializers=deserializers) diff --git a/nova/compute/api.py b/nova/compute/api.py index b0949a729..a7ea88d51 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -34,6 +34,7 @@ from nova import utils from nova import volume from nova.compute import instance_types from nova.compute import power_state +from nova.compute.utils import terminate_volumes from nova.scheduler import api as scheduler_api from nova.db import base @@ -52,6 +53,18 @@ def generate_default_hostname(instance_id): return str(instance_id) +def _is_able_to_shutdown(instance, instance_id): + states = {'terminating': "Instance %s is already being terminated", + 'migrating': "Instance %s is being migrated", + 'stopping': "Instance %s is being stopped"} + msg = states.get(instance['state_description']) + if msg: + LOG.warning(_(msg), instance_id) + return False + + return True + + class API(base.Base): """API for interacting with the compute manager.""" @@ -134,7 +147,8 @@ class API(base.Base): display_name='', display_description='', key_name=None, key_data=None, security_group='default', availability_zone=None, user_data=None, metadata={}, - injected_files=None, admin_password=None, zone_blob=None): + injected_files=None, admin_password=None, zone_blob=None, + reservation_id=None): """Verify all the input parameters regardless of the provisioning strategy being performed.""" @@ -164,6 +178,12 @@ class API(base.Base): os_type = None if 'properties' in image and 'os_type' in image['properties']: os_type = image['properties']['os_type'] + architecture = None + if 'properties' in image and 'arch' in image['properties']: + architecture = image['properties']['arch'] + vm_mode = None + if 'properties' in image and 'vm_mode' in image['properties']: + vm_mode = image['properties']['vm_mode'] if kernel_id is None: kernel_id = image['properties'].get('kernel_id', None) @@ -200,8 +220,11 @@ class API(base.Base): key_pair = db.key_pair_get(context, context.user_id, key_name) key_data = key_pair['public_key'] + if reservation_id is None: + reservation_id = utils.generate_uid('r') + base_options = { - 'reservation_id': utils.generate_uid('r'), + 'reservation_id': reservation_id, 'image_ref': image_href, 'kernel_id': kernel_id or '', 'ramdisk_id': ramdisk_id or '', @@ -222,12 +245,14 @@ class API(base.Base): 'locked': False, 'metadata': metadata, 'availability_zone': availability_zone, - 'os_type': os_type} + 'os_type': os_type, + 'architecture': architecture, + 'vm_mode': vm_mode} return (num_instances, base_options, security_groups) def create_db_entry_for_new_instance(self, context, base_options, - security_groups, num=1): + security_groups, block_device_mapping, num=1): """Create an entry in the DB for this new instance, including any related table updates (such as security groups, MAC address, etc). This will called by create() @@ -247,6 +272,23 @@ class API(base.Base): instance_id, security_group_id) + # NOTE(yamahata) + # tell vm driver to attach volume at boot time by updating + # BlockDeviceMapping + for bdm in block_device_mapping: + LOG.debug(_('bdm %s'), bdm) + assert 'device_name' in bdm + values = { + 'instance_id': instance_id, + 'device_name': bdm['device_name'], + 'delete_on_termination': bdm.get('delete_on_termination'), + 'virtual_name': bdm.get('virtual_name'), + 'snapshot_id': bdm.get('snapshot_id'), + 'volume_id': bdm.get('volume_id'), + 'volume_size': bdm.get('volume_size'), + 'no_device': bdm.get('no_device')} + self.db.block_device_mapping_create(elevated, values) + # Set sane defaults if not specified updates = dict(hostname=self.hostname_factory(instance_id)) if (not hasattr(instance, 'display_name') or @@ -281,7 +323,7 @@ class API(base.Base): 'instance_type': instance_type, 'filter': filter_class, 'blob': zone_blob, - 'num_instances': num_instances + 'num_instances': num_instances, } rpc.cast(context, @@ -300,7 +342,8 @@ class API(base.Base): display_name='', display_description='', key_name=None, key_data=None, security_group='default', availability_zone=None, user_data=None, metadata={}, - injected_files=None, admin_password=None, zone_blob=None): + injected_files=None, admin_password=None, zone_blob=None, + reservation_id=None): """Provision the instances by passing the whole request to the Scheduler for execution. Returns a Reservation ID related to the creation of all of these instances.""" @@ -312,7 +355,8 @@ class API(base.Base): display_name, display_description, key_name, key_data, security_group, availability_zone, user_data, metadata, - injected_files, admin_password, zone_blob) + injected_files, admin_password, zone_blob, + reservation_id) self._ask_scheduler_to_create_instance(context, base_options, instance_type, zone_blob, @@ -328,7 +372,8 @@ class API(base.Base): display_name='', display_description='', key_name=None, key_data=None, security_group='default', availability_zone=None, user_data=None, metadata={}, - injected_files=None, admin_password=None, zone_blob=None): + injected_files=None, admin_password=None, zone_blob=None, + reservation_id=None, block_device_mapping=None): """ Provision the instances by sending off a series of single instance requests to the Schedulers. This is fine for trival @@ -346,13 +391,16 @@ class API(base.Base): display_name, display_description, key_name, key_data, security_group, availability_zone, user_data, metadata, - injected_files, admin_password, zone_blob) + injected_files, admin_password, zone_blob, + reservation_id) + block_device_mapping = block_device_mapping or [] instances = [] LOG.debug(_("Going to run %s instances..."), num_instances) for num in range(num_instances): instance = self.create_db_entry_for_new_instance(context, - base_options, security_groups, num=num) + base_options, security_groups, + block_device_mapping, num=num) instances.append(instance) instance_id = instance['id'] @@ -462,24 +510,22 @@ class API(base.Base): rv = self.db.instance_update(context, instance_id, kwargs) return dict(rv.iteritems()) - @scheduler_api.reroute_compute("delete") - def delete(self, context, instance_id): - """Terminate an instance.""" - LOG.debug(_("Going to try to terminate %s"), instance_id) + def _get_instance(self, context, instance_id, action_str): try: - instance = self.get(context, instance_id) + return self.get(context, instance_id) except exception.NotFound: - LOG.warning(_("Instance %s was not found during terminate"), - instance_id) + LOG.warning(_("Instance %(instance_id)s was not found during " + "%(action_str)s") % + {'instance_id': instance_id, 'action_str': action_str}) raise - if instance['state_description'] == 'terminating': - LOG.warning(_("Instance %s is already being terminated"), - instance_id) - return + @scheduler_api.reroute_compute("delete") + def delete(self, context, instance_id): + """Terminate an instance.""" + LOG.debug(_("Going to try to terminate %s"), instance_id) + instance = self._get_instance(context, instance_id, 'terminating') - if instance['state_description'] == 'migrating': - LOG.warning(_("Instance %s is being migrated"), instance_id) + if not _is_able_to_shutdown(instance, instance_id): return self.update(context, @@ -493,12 +539,59 @@ class API(base.Base): self._cast_compute_message('terminate_instance', context, instance_id, host) else: + terminate_volumes(self.db, context, instance_id) self.db.instance_destroy(context, instance_id) + @scheduler_api.reroute_compute("stop") + def stop(self, context, instance_id): + """Stop an instance.""" + LOG.debug(_("Going to try to stop %s"), instance_id) + + instance = self._get_instance(context, instance_id, 'stopping') + if not _is_able_to_shutdown(instance, instance_id): + return + + self.update(context, + instance['id'], + state_description='stopping', + state=power_state.NOSTATE, + terminated_at=utils.utcnow()) + + host = instance['host'] + if host: + self._cast_compute_message('stop_instance', context, + instance_id, host) + + def start(self, context, instance_id): + """Start an instance.""" + LOG.debug(_("Going to try to start %s"), instance_id) + instance = self._get_instance(context, instance_id, 'starting') + if instance['state_description'] != 'stopped': + _state_description = instance['state_description'] + LOG.warning(_("Instance %(instance_id)s is not " + "stopped(%(_state_description)s)") % locals()) + return + + # TODO(yamahata): injected_files isn't supported right now. + # It is used only for osapi. not for ec2 api. + # availability_zone isn't used by run_instance. + rpc.cast(context, + FLAGS.scheduler_topic, + {"method": "start_instance", + "args": {"topic": FLAGS.compute_topic, + "instance_id": instance_id}}) + def get(self, context, instance_id): """Get a single instance with the given instance_id.""" - rv = self.db.instance_get(context, instance_id) - return dict(rv.iteritems()) + # NOTE(sirp): id used to be exclusively integer IDs; now we're + # accepting both UUIDs and integer IDs. The handling of this + # is done in db/sqlalchemy/api/instance_get + if utils.is_uuid_like(instance_id): + uuid = instance_id + instance = self.db.instance_get_by_uuid(context, uuid) + else: + instance = self.db.instance_get(context, instance_id) + return dict(instance.iteritems()) @scheduler_api.reroute_compute("get") def routing_get(self, context, instance_id): @@ -510,6 +603,25 @@ class API(base.Base): """ return self.get(context, instance_id) + def get_all_across_zones(self, context, reservation_id): + """Get all instances with this reservation_id, across + all available Zones (if any). + """ + context = context.elevated() + instances = self.db.instance_get_all_by_reservation( + context, reservation_id) + + children = scheduler_api.call_zone_method(context, "list", + novaclient_collection_name="servers", + reservation_id=reservation_id) + + for zone, servers in children: + for server in servers: + # Results are ready to send to user. No need to scrub. + server._info['_is_precooked'] = True + instances.append(server._info) + return instances + def get_all(self, context, project_id=None, reservation_id=None, fixed_ip=None): """Get all instances filtered by one of the given parameters. @@ -518,8 +630,7 @@ class API(base.Base): all instances in the system. """ if reservation_id is not None: - return self.db.instance_get_all_by_reservation( - context, reservation_id) + return self.get_all_across_zones(context, reservation_id) if fixed_ip is not None: return self.db.fixed_ip_get_instance(context, fixed_ip) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 245958de7..54eaf2082 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -53,6 +53,7 @@ from nova import rpc from nova import utils from nova import volume from nova.compute import power_state +from nova.compute.utils import terminate_volumes from nova.virt import driver @@ -214,8 +215,63 @@ class ComputeManager(manager.SchedulerDependentManager): """ return self.driver.refresh_security_group_members(security_group_id) - @exception.wrap_exception - def run_instance(self, context, instance_id, **kwargs): + def _setup_block_device_mapping(self, context, instance_id): + """setup volumes for block device mapping""" + self.db.instance_set_state(context, + instance_id, + power_state.NOSTATE, + 'block_device_mapping') + + volume_api = volume.API() + block_device_mapping = [] + for bdm in self.db.block_device_mapping_get_all_by_instance( + context, instance_id): + LOG.debug(_("setting up bdm %s"), bdm) + if ((bdm['snapshot_id'] is not None) and + (bdm['volume_id'] is None)): + # TODO(yamahata): default name and description + vol = volume_api.create(context, bdm['volume_size'], + bdm['snapshot_id'], '', '') + # TODO(yamahata): creating volume simultaneously + # reduces creation time? + volume_api.wait_creation(context, vol['id']) + self.db.block_device_mapping_update( + context, bdm['id'], {'volume_id': vol['id']}) + bdm['volume_id'] = vol['id'] + + if not ((bdm['snapshot_id'] is None) or + (bdm['volume_id'] is not None)): + LOG.error(_('corrupted state of block device mapping ' + 'id: %(id)s ' + 'snapshot: %(snapshot_id) volume: %(vollume_id)') % + {'id': bdm['id'], + 'snapshot_id': bdm['snapshot'], + 'volume_id': bdm['volume_id']}) + raise exception.ApiError(_('broken block device mapping %d') % + bdm['id']) + + if bdm['volume_id'] is not None: + volume_api.check_attach(context, + volume_id=bdm['volume_id']) + dev_path = self._attach_volume_boot(context, instance_id, + bdm['volume_id'], + bdm['device_name']) + block_device_mapping.append({'device_path': dev_path, + 'mount_device': + bdm['device_name']}) + elif bdm['virtual_name'] is not None: + # TODO(yamahata): ephemeral/swap device support + LOG.debug(_('block_device_mapping: ' + 'ephemeral device is not supported yet')) + else: + # TODO(yamahata): NoDevice support + assert bdm['no_device'] + LOG.debug(_('block_device_mapping: ' + 'no device is not supported yet')) + + return block_device_mapping + + def _run_instance(self, context, instance_id, **kwargs): """Launch a new instance with specified options.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) @@ -235,54 +291,79 @@ class ComputeManager(manager.SchedulerDependentManager): 'networking') is_vpn = instance_ref['image_ref'] == str(FLAGS.vpn_image_id) - # NOTE(vish): This could be a cast because we don't do anything - # with the address currently, but I'm leaving it as - # a call to ensure that network setup completes. We - # will eventually also need to save the address here. - if not FLAGS.stub_network: - address = rpc.call(context, - self.get_network_topic(context), - {"method": "allocate_fixed_ip", - "args": {"instance_id": instance_id, - "vpn": is_vpn}}) - - self.network_manager.setup_compute_network(context, - instance_id) - - # TODO(vish) check to make sure the availability zone matches - self._update_state(context, instance_id, power_state.BUILDING) - try: - self.driver.spawn(instance_ref) - except Exception as ex: # pylint: disable=W0702 - msg = _("Instance '%(instance_id)s' failed to spawn. Is " - "virtualization enabled in the BIOS? Details: " - "%(ex)s") % locals() - LOG.exception(msg) - - if not FLAGS.stub_network and FLAGS.auto_assign_floating_ip: - public_ip = self.network_api.allocate_floating_ip(context) - - self.db.floating_ip_set_auto_assigned(context, public_ip) - fixed_ip = self.db.fixed_ip_get_by_address(context, address) - floating_ip = self.db.floating_ip_get_by_address(context, - public_ip) - - self.network_api.associate_floating_ip(context, - floating_ip, - fixed_ip, - affect_auto_assigned=True) + # NOTE(vish): This could be a cast because we don't do anything + # with the address currently, but I'm leaving it as + # a call to ensure that network setup completes. We + # will eventually also need to save the address here. + if not FLAGS.stub_network: + address = rpc.call(context, + self.get_network_topic(context), + {"method": "allocate_fixed_ip", + "args": {"instance_id": instance_id, + "vpn": is_vpn}}) - self._update_launched_at(context, instance_id) - self._update_state(context, instance_id) + self.network_manager.setup_compute_network(context, + instance_id) + + block_device_mapping = self._setup_block_device_mapping( + context, + instance_id) + + # TODO(vish) check to make sure the availability zone matches + self._update_state(context, instance_id, power_state.BUILDING) + + try: + self.driver.spawn(instance_ref, + block_device_mapping=block_device_mapping) + except Exception as ex: # pylint: disable=W0702 + msg = _("Instance '%(instance_id)s' failed to spawn. Is " + "virtualization enabled in the BIOS? Details: " + "%(ex)s") % locals() + LOG.exception(msg) + + if not FLAGS.stub_network and FLAGS.auto_assign_floating_ip: + public_ip = self.network_api.allocate_floating_ip(context) + + self.db.floating_ip_set_auto_assigned(context, public_ip) + fixed_ip = self.db.fixed_ip_get_by_address(context, address) + floating_ip = self.db.floating_ip_get_by_address(context, + public_ip) + + self.network_api.associate_floating_ip( + context, + floating_ip, + fixed_ip, + affect_auto_assigned=True) + + self._update_launched_at(context, instance_id) + self._update_state(context, instance_id) + except exception.InstanceNotFound: + # FIXME(wwolf): We are just ignoring InstanceNotFound + # exceptions here in case the instance was immediately + # deleted before it actually got created. This should + # be fixed once we have no-db-messaging + pass + + @exception.wrap_exception + def run_instance(self, context, instance_id, **kwargs): + self._run_instance(context, instance_id, **kwargs) @exception.wrap_exception @checks_instance_lock - def terminate_instance(self, context, instance_id): - """Terminate an instance on this host.""" + def start_instance(self, context, instance_id): + """Starting an instance on this host.""" + # TODO(yamahata): injected_files isn't supported. + # Anyway OSAPI doesn't support stop/start yet + self._run_instance(context, instance_id) + + def _shutdown_instance(self, context, instance_id, action_str): + """Shutdown an instance on this host.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) - LOG.audit(_("Terminating instance %s"), instance_id, context=context) + LOG.audit(_("%(action_str)s instance %(instance_id)s") % + {'action_str': action_str, 'instance_id': instance_id}, + context=context) fixed_ip = instance_ref.get('fixed_ip') if not FLAGS.stub_network and fixed_ip: @@ -318,18 +399,36 @@ class ComputeManager(manager.SchedulerDependentManager): volumes = instance_ref.get('volumes') or [] for volume in volumes: - self.detach_volume(context, instance_id, volume['id']) - if instance_ref['state'] == power_state.SHUTOFF: + self._detach_volume(context, instance_id, volume['id'], False) + + if (instance_ref['state'] == power_state.SHUTOFF and + instance_ref['state_description'] != 'stopped'): self.db.instance_destroy(context, instance_id) raise exception.Error(_('trying to destroy already destroyed' ' instance: %s') % instance_id) self.driver.destroy(instance_ref) + if action_str == 'Terminating': + terminate_volumes(self.db, context, instance_id) + + @exception.wrap_exception + @checks_instance_lock + def terminate_instance(self, context, instance_id): + """Terminate an instance on this host.""" + self._shutdown_instance(context, instance_id, 'Terminating') + # TODO(ja): should we keep it in a terminated state for a bit? self.db.instance_destroy(context, instance_id) @exception.wrap_exception @checks_instance_lock + def stop_instance(self, context, instance_id): + """Stopping an instance on this host.""" + self._shutdown_instance(context, instance_id, 'Stopping') + # instance state will be updated to stopped by _poll_instance_states() + + @exception.wrap_exception + @checks_instance_lock def rebuild_instance(self, context, instance_id, **kwargs): """Destroy and re-make this instance. @@ -472,6 +571,24 @@ class ComputeManager(manager.SchedulerDependentManager): @exception.wrap_exception @checks_instance_lock + def agent_update(self, context, instance_id, url, md5hash): + """Update agent running on an instance on this host.""" + context = context.elevated() + instance_ref = self.db.instance_get(context, instance_id) + instance_id = instance_ref['id'] + instance_state = instance_ref['state'] + expected_state = power_state.RUNNING + if instance_state != expected_state: + LOG.warn(_('trying to update agent on a non-running ' + 'instance: %(instance_id)s (state: %(instance_state)s ' + 'expected: %(expected_state)s)') % locals()) + nm = instance_ref['name'] + msg = _('instance %(nm)s: updating agent to %(url)s') % locals() + LOG.audit(msg) + self.driver.agent_update(instance_ref, url, md5hash) + + @exception.wrap_exception + @checks_instance_lock def rescue_instance(self, context, instance_id): """Rescue an instance on this host.""" context = context.elevated() @@ -693,7 +810,6 @@ class ComputeManager(manager.SchedulerDependentManager): def get_diagnostics(self, context, instance_id): """Retrieve diagnostics for an instance on this host.""" instance_ref = self.db.instance_get(context, instance_id) - if instance_ref["state"] == power_state.RUNNING: LOG.audit(_("instance %s: retrieving diagnostics"), instance_id, context=context) @@ -800,6 +916,22 @@ class ComputeManager(manager.SchedulerDependentManager): instance_ref = self.db.instance_get(context, instance_id) return self.driver.get_vnc_console(instance_ref) + def _attach_volume_boot(self, context, instance_id, volume_id, mountpoint): + """Attach a volume to an instance at boot time. So actual attach + is done by instance creation""" + + # TODO(yamahata): + # should move check_attach to volume manager? + volume.API().check_attach(context, volume_id) + + context = context.elevated() + LOG.audit(_("instance %(instance_id)s: booting with " + "volume %(volume_id)s at %(mountpoint)s") % + locals(), context=context) + dev_path = self.volume_manager.setup_compute_volume(context, volume_id) + self.db.volume_attached(context, volume_id, instance_id, mountpoint) + return dev_path + @checks_instance_lock def attach_volume(self, context, instance_id, volume_id, mountpoint): """Attach a volume to an instance.""" @@ -817,6 +949,16 @@ class ComputeManager(manager.SchedulerDependentManager): volume_id, instance_id, mountpoint) + values = { + 'instance_id': instance_id, + 'device_name': mountpoint, + 'delete_on_termination': False, + 'virtual_name': None, + 'snapshot_id': None, + 'volume_id': volume_id, + 'volume_size': None, + 'no_device': None} + self.db.block_device_mapping_create(context, values) except Exception as exc: # pylint: disable=W0702 # NOTE(vish): The inline callback eats the exception info so we # log the traceback here and reraise the same @@ -831,7 +973,7 @@ class ComputeManager(manager.SchedulerDependentManager): @exception.wrap_exception @checks_instance_lock - def detach_volume(self, context, instance_id, volume_id): + def _detach_volume(self, context, instance_id, volume_id, destroy_bdm): """Detach a volume from an instance.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) @@ -847,8 +989,15 @@ class ComputeManager(manager.SchedulerDependentManager): volume_ref['mountpoint']) self.volume_manager.remove_compute_volume(context, volume_id) self.db.volume_detached(context, volume_id) + if destroy_bdm: + self.db.block_device_mapping_destroy_by_instance_and_volume( + context, instance_id, volume_id) return True + def detach_volume(self, context, instance_id, volume_id): + """Detach a volume from an instance.""" + return self._detach_volume(context, instance_id, volume_id, True) + def remove_volume(self, context, volume_id): """Remove volume on compute host. @@ -1174,11 +1323,14 @@ class ComputeManager(manager.SchedulerDependentManager): "State=%(db_state)s, so setting state to " "shutoff.") % locals()) vm_state = power_state.SHUTOFF + if db_instance['state_description'] == 'stopping': + self.db.instance_stop(context, db_instance['id']) + continue else: vm_state = vm_instance.state vms_not_found_in_db.remove(name) - if db_instance['state_description'] == 'migrating': + if (db_instance['state_description'] in ['migrating', 'stopping']): # A situation which db record exists, but no instance" # sometimes occurs while live-migration at src compute, # this case should be ignored. diff --git a/nova/compute/utils.py b/nova/compute/utils.py new file mode 100644 index 000000000..c8cb9bab8 --- /dev/null +++ b/nova/compute/utils.py @@ -0,0 +1,29 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 VA Linux Systems Japan K.K +# Copyright (c) 2011 Isaku Yamahata +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import volume + + +def terminate_volumes(db, context, instance_id): + """delete volumes of delete_on_termination=True in block device mapping""" + volume_api = volume.API() + for bdm in db.block_device_mapping_get_all_by_instance(context, + instance_id): + #LOG.debug(_("terminating bdm %s") % bdm) + if bdm['volume_id'] and bdm['delete_on_termination']: + volume_api.delete(context, bdm['volume_id']) + db.block_device_mapping_destroy(context, bdm['id']) diff --git a/nova/crypto.py b/nova/crypto.py index bdc32482a..8d535f426 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -176,7 +176,8 @@ def revoke_certs_by_project(project_id): def revoke_certs_by_user_and_project(user_id, project_id): """Revoke certs for user in project.""" admin = context.get_admin_context() - for cert in db.certificate_get_all_by_user(admin, user_id, project_id): + for cert in db.certificate_get_all_by_user_and_project(admin, + user_id, project_id): revoke_cert(cert['project_id'], cert['file_name']) diff --git a/nova/db/api.py b/nova/db/api.py index 4e0aa60a2..8f8e856b8 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -414,6 +414,16 @@ def instance_destroy(context, instance_id): return IMPL.instance_destroy(context, instance_id) +def instance_stop(context, instance_id): + """Stop the instance or raise if it does not exist.""" + return IMPL.instance_stop(context, instance_id) + + +def instance_get_by_uuid(context, uuid): + """Get an instance or raise if it does not exist.""" + return IMPL.instance_get_by_uuid(context, uuid) + + def instance_get(context, instance_id): """Get an instance or raise if it does not exist.""" return IMPL.instance_get(context, instance_id) @@ -920,6 +930,36 @@ def snapshot_update(context, snapshot_id, values): #################### +def block_device_mapping_create(context, values): + """Create an entry of block device mapping""" + return IMPL.block_device_mapping_create(context, values) + + +def block_device_mapping_update(context, bdm_id, values): + """Create an entry of block device mapping""" + return IMPL.block_device_mapping_update(context, bdm_id, values) + + +def block_device_mapping_get_all_by_instance(context, instance_id): + """Get all block device mapping belonging to a instance""" + return IMPL.block_device_mapping_get_all_by_instance(context, instance_id) + + +def block_device_mapping_destroy(context, bdm_id): + """Destroy the block device mapping.""" + return IMPL.block_device_mapping_destroy(context, bdm_id) + + +def block_device_mapping_destroy_by_instance_and_volume(context, instance_id, + volume_id): + """Destroy the block device mapping or raise if it does not exist.""" + return IMPL.block_device_mapping_destroy_by_instance_and_volume( + context, instance_id, volume_id) + + +#################### + + def security_group_get_all(context): """Get all security groups.""" return IMPL.security_group_get_all(context) @@ -1247,3 +1287,32 @@ def instance_metadata_delete(context, instance_id, key): def instance_metadata_update_or_create(context, instance_id, metadata): """Create or update instance metadata.""" IMPL.instance_metadata_update_or_create(context, instance_id, metadata) + + +#################### + + +def agent_build_create(context, values): + """Create a new agent build entry.""" + return IMPL.agent_build_create(context, values) + + +def agent_build_get_by_triple(context, hypervisor, os, architecture): + """Get agent build by hypervisor/OS/architecture triple.""" + return IMPL.agent_build_get_by_triple(context, hypervisor, os, + architecture) + + +def agent_build_get_all(context): + """Get all agent builds.""" + return IMPL.agent_build_get_all(context) + + +def agent_build_destroy(context, agent_update_id): + """Destroy agent build entry.""" + IMPL.agent_build_destroy(context, agent_update_id) + + +def agent_build_update(context, agent_build_id, values): + """Update agent build entry.""" + IMPL.agent_build_update(context, agent_build_id, values) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 73870d2f3..a2500a38d 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -18,7 +18,7 @@ """ Implementation of SQLAlchemy backend. """ - +import traceback import warnings from nova import db @@ -797,6 +797,8 @@ def instance_create(context, values): values['metadata'] = _metadata_refs(values.get('metadata')) instance_ref = models.Instance() + instance_ref['uuid'] = str(utils.gen_uuid()) + instance_ref.update(values) session = get_session() @@ -840,37 +842,65 @@ def instance_destroy(context, instance_id): @require_context +def instance_stop(context, instance_id): + session = get_session() + with session.begin(): + from nova.compute import power_state + session.query(models.Instance).\ + filter_by(id=instance_id).\ + update({'host': None, + 'state': power_state.SHUTOFF, + 'state_description': 'stopped', + 'updated_at': literal_column('updated_at')}) + session.query(models.SecurityGroupInstanceAssociation).\ + filter_by(instance_id=instance_id).\ + update({'updated_at': literal_column('updated_at')}) + session.query(models.InstanceMetadata).\ + filter_by(instance_id=instance_id).\ + update({'updated_at': literal_column('updated_at')}) + + +@require_context +def instance_get_by_uuid(context, uuid, session=None): + partial = _build_instance_get(context, session=session) + result = partial.filter_by(uuid=uuid) + result = result.first() + if not result: + # FIXME(sirp): it would be nice if InstanceNotFound would accept a + # uuid parameter as well + raise exception.InstanceNotFound(instance_id=uuid) + return result + + +@require_context def instance_get(context, instance_id, session=None): + partial = _build_instance_get(context, session=session) + result = partial.filter_by(id=instance_id) + result = result.first() + if not result: + raise exception.InstanceNotFound(instance_id=instance_id) + return result + + +@require_context +def _build_instance_get(context, session=None): if not session: session = get_session() - result = None + + partial = session.query(models.Instance).\ + options(joinedload_all('fixed_ip.floating_ips')).\ + options(joinedload_all('security_groups.rules')).\ + options(joinedload('volumes')).\ + options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ + options(joinedload('instance_type')) if is_admin_context(context): - result = session.query(models.Instance).\ - options(joinedload_all('fixed_ip.floating_ips')).\ - options(joinedload_all('security_groups.rules')).\ - options(joinedload('volumes')).\ - options(joinedload_all('fixed_ip.network')).\ - options(joinedload('metadata')).\ - options(joinedload('instance_type')).\ - filter_by(id=instance_id).\ - filter_by(deleted=can_read_deleted(context)).\ - first() + partial = partial.filter_by(deleted=can_read_deleted(context)) elif is_user_context(context): - result = session.query(models.Instance).\ - options(joinedload_all('fixed_ip.floating_ips')).\ - options(joinedload_all('security_groups.rules')).\ - options(joinedload('volumes')).\ - options(joinedload('metadata')).\ - options(joinedload('instance_type')).\ - filter_by(project_id=context.project_id).\ - filter_by(id=instance_id).\ - filter_by(deleted=False).\ - first() - if not result: - raise exception.InstanceNotFound(instance_id=instance_id) - - return result + partial = partial.filter_by(project_id=context.project_id).\ + filter_by(deleted=False) + return partial @require_admin_context @@ -907,6 +937,7 @@ def instance_get_all_by_host(context, host): options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload('security_groups')).\ options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ options(joinedload('instance_type')).\ filter_by(host=host).\ filter_by(deleted=can_read_deleted(context)).\ @@ -922,6 +953,7 @@ def instance_get_all_by_project(context, project_id): options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload('security_groups')).\ options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ options(joinedload('instance_type')).\ filter_by(project_id=project_id).\ filter_by(deleted=can_read_deleted(context)).\ @@ -937,6 +969,7 @@ def instance_get_all_by_reservation(context, reservation_id): options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload('security_groups')).\ options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ options(joinedload('instance_type')).\ filter_by(reservation_id=reservation_id).\ filter_by(deleted=can_read_deleted(context)).\ @@ -946,6 +979,7 @@ def instance_get_all_by_reservation(context, reservation_id): options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload('security_groups')).\ options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ options(joinedload('instance_type')).\ filter_by(project_id=context.project_id).\ filter_by(reservation_id=reservation_id).\ @@ -959,6 +993,8 @@ def instance_get_project_vpn(context, project_id): return session.query(models.Instance).\ options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload('security_groups')).\ + options(joinedload_all('fixed_ip.network')).\ + options(joinedload('metadata')).\ options(joinedload('instance_type')).\ filter_by(project_id=project_id).\ filter_by(image_ref=str(FLAGS.vpn_image_id)).\ @@ -1877,6 +1913,66 @@ def snapshot_update(context, snapshot_id, values): @require_context +def block_device_mapping_create(context, values): + bdm_ref = models.BlockDeviceMapping() + bdm_ref.update(values) + + session = get_session() + with session.begin(): + bdm_ref.save(session=session) + + +@require_context +def block_device_mapping_update(context, bdm_id, values): + session = get_session() + with session.begin(): + session.query(models.BlockDeviceMapping).\ + filter_by(id=bdm_id).\ + filter_by(deleted=False).\ + update(values) + + +@require_context +def block_device_mapping_get_all_by_instance(context, instance_id): + session = get_session() + result = session.query(models.BlockDeviceMapping).\ + filter_by(instance_id=instance_id).\ + filter_by(deleted=False).\ + all() + if not result: + return [] + return result + + +@require_context +def block_device_mapping_destroy(context, bdm_id): + session = get_session() + with session.begin(): + session.query(models.BlockDeviceMapping).\ + filter_by(id=bdm_id).\ + update({'deleted': True, + 'deleted_at': utils.utcnow(), + 'updated_at': literal_column('updated_at')}) + + +@require_context +def block_device_mapping_destroy_by_instance_and_volume(context, instance_id, + volume_id): + session = get_session() + with session.begin(): + session.query(models.BlockDeviceMapping).\ + filter_by(instance_id=instance_id).\ + filter_by(volume_id=volume_id).\ + filter_by(deleted=False).\ + update({'deleted': True, + 'deleted_at': utils.utcnow(), + 'updated_at': literal_column('updated_at')}) + + +################### + + +@require_context def security_group_get_all(context): session = get_session() return session.query(models.SecurityGroup).\ @@ -2609,7 +2705,17 @@ def zone_get_all(context): #################### + +def require_instance_exists(func): + def new_func(context, instance_id, *args, **kwargs): + db.api.instance_get(context, instance_id) + return func(context, instance_id, *args, **kwargs) + new_func.__name__ = func.__name__ + return new_func + + @require_context +@require_instance_exists def instance_metadata_get(context, instance_id): session = get_session() @@ -2625,6 +2731,7 @@ def instance_metadata_get(context, instance_id): @require_context +@require_instance_exists def instance_metadata_delete(context, instance_id, key): session = get_session() session.query(models.InstanceMetadata).\ @@ -2637,6 +2744,7 @@ def instance_metadata_delete(context, instance_id, key): @require_context +@require_instance_exists def instance_metadata_delete_all(context, instance_id): session = get_session() session.query(models.InstanceMetadata).\ @@ -2648,6 +2756,7 @@ def instance_metadata_delete_all(context, instance_id): @require_context +@require_instance_exists def instance_metadata_get_item(context, instance_id, key): session = get_session() @@ -2664,6 +2773,7 @@ def instance_metadata_get_item(context, instance_id, key): @require_context +@require_instance_exists def instance_metadata_update_or_create(context, instance_id, metadata): session = get_session() @@ -2682,3 +2792,54 @@ def instance_metadata_update_or_create(context, instance_id, metadata): meta_ref.save(session=session) return metadata + + +@require_admin_context +def agent_build_create(context, values): + agent_build_ref = models.AgentBuild() + agent_build_ref.update(values) + agent_build_ref.save() + return agent_build_ref + + +@require_admin_context +def agent_build_get_by_triple(context, hypervisor, os, architecture, + session=None): + if not session: + session = get_session() + return session.query(models.AgentBuild).\ + filter_by(hypervisor=hypervisor).\ + filter_by(os=os).\ + filter_by(architecture=architecture).\ + filter_by(deleted=False).\ + first() + + +@require_admin_context +def agent_build_get_all(context): + session = get_session() + return session.query(models.AgentBuild).\ + filter_by(deleted=False).\ + all() + + +@require_admin_context +def agent_build_destroy(context, agent_build_id): + session = get_session() + with session.begin(): + session.query(models.AgentBuild).\ + filter_by(id=agent_build_id).\ + update({'deleted': 1, + 'deleted_at': datetime.datetime.utcnow(), + 'updated_at': literal_column('updated_at')}) + + +@require_admin_context +def agent_build_update(context, agent_build_id, values): + session = get_session() + with session.begin(): + agent_build_ref = session.query(models.AgentBuild).\ + filter_by(id=agent_build_id). \ + first() + agent_build_ref.update(values) + agent_build_ref.save(session=session) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/023_add_vm_mode_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/023_add_vm_mode_to_instances.py new file mode 100644 index 000000000..0c587f569 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/023_add_vm_mode_to_instances.py @@ -0,0 +1,45 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Column, Integer, MetaData, String, Table + +meta = MetaData() + +instances_vm_mode = Column('vm_mode', + String(length=255, convert_unicode=False, + assert_unicode=None, unicode_error=None, + _warn_on_bytestring=False), + nullable=True) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + instances = Table('instances', meta, autoload=True, + autoload_with=migrate_engine) + + instances.create_column(instances_vm_mode) + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + + instances = Table('instances', meta, autoload=True, + autoload_with=migrate_engine) + + instances.drop_column('vm_mode') diff --git a/nova/db/sqlalchemy/migrate_repo/versions/024_add_block_device_mapping.py b/nova/db/sqlalchemy/migrate_repo/versions/024_add_block_device_mapping.py new file mode 100644 index 000000000..6e9b806cb --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/024_add_block_device_mapping.py @@ -0,0 +1,87 @@ +# Copyright 2011 OpenStack LLC. +# Copyright 2011 Isaku Yamahata +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import MetaData, Table, Column +from sqlalchemy import DateTime, Boolean, Integer, String +from sqlalchemy import ForeignKey +from nova import log as logging + +meta = MetaData() + +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +volumes = Table('volumes', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + +snapshots = Table('snapshots', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +block_device_mapping = Table('block_device_mapping', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, autoincrement=True), + Column('instance_id', + Integer(), + ForeignKey('instances.id'), + nullable=False), + Column('device_name', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=False), + Column('delete_on_termination', + Boolean(create_constraint=True, name=None), + default=False), + Column('virtual_name', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + nullable=True), + Column('snapshot_id', + Integer(), + ForeignKey('snapshots.id'), + nullable=True), + Column('volume_id', Integer(), ForeignKey('volumes.id'), + nullable=True), + Column('volume_size', Integer(), nullable=True), + Column('no_device', + Boolean(create_constraint=True, name=None), + nullable=True), + ) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + try: + block_device_mapping.create() + except Exception: + logging.info(repr(block_device_mapping)) + logging.exception('Exception while creating table') + meta.drop_all(tables=[block_device_mapping]) + raise + + +def downgrade(migrate_engine): + # Operations to reverse the above upgrade go here. + block_device_mapping.drop() diff --git a/nova/db/sqlalchemy/migrate_repo/versions/025_add_uuid_to_instances.py b/nova/db/sqlalchemy/migrate_repo/versions/025_add_uuid_to_instances.py new file mode 100644 index 000000000..27f30d536 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/025_add_uuid_to_instances.py @@ -0,0 +1,43 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 OpenStack LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Column, Integer, MetaData, String, Table + +from nova import utils + + +meta = MetaData() + +instances = Table("instances", meta, + Column("id", Integer(), primary_key=True, nullable=False)) +uuid_column = Column("uuid", String(36)) + + +def upgrade(migrate_engine): + meta.bind = migrate_engine + instances.create_column(uuid_column) + + rows = migrate_engine.execute(instances.select()) + for row in rows: + instance_uuid = str(utils.gen_uuid()) + migrate_engine.execute(instances.update()\ + .where(instances.c.id == row[0])\ + .values(uuid=instance_uuid)) + + +def downgrade(migrate_engine): + meta.bind = migrate_engine + instances.drop_column(uuid_column) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/026_add_agent_table.py b/nova/db/sqlalchemy/migrate_repo/versions/026_add_agent_table.py new file mode 100644 index 000000000..640e96138 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/026_add_agent_table.py @@ -0,0 +1,73 @@ +# Copyright 2011 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import Boolean, Column, DateTime, Integer +from sqlalchemy import MetaData, String, Table +from nova import log as logging + +meta = MetaData() + +# +# New Tables +# +builds = Table('agent_builds', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('hypervisor', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('os', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('architecture', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('version', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('url', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('md5hash', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + + +# +# New Column +# + +architecture = Column('architecture', String(length=255)) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (builds, ): + try: + table.create() + except Exception: + logging.info(repr(table)) + + instances = Table('instances', meta, autoload=True, + autoload_with=migrate_engine) + + # Add columns to existing tables + instances.create_column(architecture) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 239f6e96a..f9375960d 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -232,6 +232,9 @@ class Instance(BASE, NovaBase): locked = Column(Boolean) os_type = Column(String(255)) + architecture = Column(String(255)) + vm_mode = Column(String(255)) + uuid = Column(String(36)) # TODO(vish): see Ewan's email about state improvements, probably # should be in a driver base class or some such @@ -356,6 +359,45 @@ class Snapshot(BASE, NovaBase): display_description = Column(String(255)) +class BlockDeviceMapping(BASE, NovaBase): + """Represents block device mapping that is defined by EC2""" + __tablename__ = "block_device_mapping" + id = Column(Integer, primary_key=True, autoincrement=True) + + instance_id = Column(Integer, ForeignKey('instances.id'), nullable=False) + instance = relationship(Instance, + backref=backref('balock_device_mapping'), + foreign_keys=instance_id, + primaryjoin='and_(BlockDeviceMapping.instance_id==' + 'Instance.id,' + 'BlockDeviceMapping.deleted==' + 'False)') + device_name = Column(String(255), nullable=False) + + # default=False for compatibility of the existing code. + # With EC2 API, + # default True for ami specified device. + # default False for created with other timing. + delete_on_termination = Column(Boolean, default=False) + + # for ephemeral device + virtual_name = Column(String(255), nullable=True) + + # for snapshot or volume + snapshot_id = Column(Integer, ForeignKey('snapshots.id'), nullable=True) + # outer join + snapshot = relationship(Snapshot, + foreign_keys=snapshot_id) + + volume_id = Column(Integer, ForeignKey('volumes.id'), nullable=True) + volume = relationship(Volume, + foreign_keys=volume_id) + volume_size = Column(Integer, nullable=True) + + # for no device to suppress devices. + no_device = Column(Boolean, nullable=True) + + class ExportDevice(BASE, NovaBase): """Represates a shelf and blade that a volume can be exported on.""" __tablename__ = 'export_devices' @@ -672,6 +714,18 @@ class Zone(BASE, NovaBase): password = Column(String(255)) +class AgentBuild(BASE, NovaBase): + """Represents an agent build.""" + __tablename__ = 'agent_builds' + id = Column(Integer, primary_key=True) + hypervisor = Column(String(255)) + os = Column(String(255)) + architecture = Column(String(255)) + version = Column(String(255)) + url = Column(String(255)) + md5hash = Column(String(255)) + + def register_models(): """Register Models and create metadata. @@ -685,7 +739,7 @@ def register_models(): Network, SecurityGroup, SecurityGroupIngressRule, SecurityGroupInstanceAssociation, AuthToken, User, Project, Certificate, ConsolePool, Console, Zone, - InstanceMetadata, Migration) + AgentBuild, InstanceMetadata, Migration) engine = create_engine(FLAGS.sql_connection, echo=False) for model in models: model.metadata.create_all(engine) diff --git a/nova/exception.py b/nova/exception.py index 1571dd032..f3a452228 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -585,3 +585,7 @@ class InstanceExists(Duplicate): class MigrationError(NovaException): message = _("Migration error") + ": %(reason)s" + + +class MalformedRequestBody(NovaException): + message = _("Malformed message body: %(reason)s") diff --git a/nova/flags.py b/nova/flags.py index acfcf8d68..57a4ecf2f 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -272,7 +272,7 @@ DEFINE_string('aws_access_key_id', 'admin', 'AWS Access ID') DEFINE_string('aws_secret_access_key', 'admin', 'AWS Access Key') # NOTE(sirp): my_ip interpolation doesn't work within nested structures DEFINE_list('glance_api_servers', - ['127.0.0.1:9292'], + ['%s:9292' % _get_my_ip()], 'list of glance api servers available to nova (host:port)') DEFINE_integer('s3_port', 3333, 's3 port') DEFINE_string('s3_host', '$my_ip', 's3 host (for infrastructure)') @@ -364,7 +364,7 @@ DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', 'Manager for scheduler') # The service to use for image search and retrieval -DEFINE_string('image_service', 'nova.image.local.LocalImageService', +DEFINE_string('image_service', 'nova.image.glance.GlanceImageService', 'The service to use for retrieving and searching for images.') DEFINE_string('host', socket.gethostname(), diff --git a/nova/image/fake.py b/nova/image/fake.py index 70a5f0e22..c4b3d5fd6 100644 --- a/nova/image/fake.py +++ b/nova/image/fake.py @@ -120,6 +120,14 @@ class _FakeImageService(service.BaseImageService): image_id, self.images) raise exception.ImageNotFound(image_id=image_id) + def show_by_name(self, context, name): + """Returns a dict containing image data for the given name.""" + images = copy.deepcopy(self.images.values()) + for image in images: + if name == image.get('name'): + return image + raise exception.ImageNotFound(image_id=name) + def create(self, context, metadata, data=None): """Store the image data and return the new image id. diff --git a/nova/image/local.py b/nova/image/local.py deleted file mode 100644 index c7dee4573..000000000 --- a/nova/image/local.py +++ /dev/null @@ -1,167 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import json -import os.path -import random -import shutil - -from nova import exception -from nova import flags -from nova import log as logging -from nova import utils -from nova.image import service - - -FLAGS = flags.FLAGS -flags.DEFINE_string('images_path', '$state_path/images', - 'path to decrypted images') - - -LOG = logging.getLogger('nova.image.local') - - -class LocalImageService(service.BaseImageService): - """Image service storing images to local disk. - - It assumes that image_ids are integers. - - """ - - def __init__(self): - self._path = FLAGS.images_path - - def _path_to(self, image_id, fname='info.json'): - if fname: - return os.path.join(self._path, '%08x' % int(image_id), fname) - return os.path.join(self._path, '%08x' % int(image_id)) - - def _ids(self): - """The list of all image ids.""" - images = [] - for image_dir in os.listdir(self._path): - try: - unhexed_image_id = int(image_dir, 16) - except ValueError: - LOG.error(_('%s is not in correct directory naming format') - % image_dir) - else: - images.append(unhexed_image_id) - return images - - def index(self, context, filters=None, marker=None, limit=None): - # TODO(blamar): Make use of filters, marker, and limit - filtered = [] - image_metas = self.detail(context) - for image_meta in image_metas: - meta = utils.subset_dict(image_meta, ('id', 'name')) - filtered.append(meta) - return filtered - - def detail(self, context, filters=None, marker=None, limit=None): - # TODO(blamar): Make use of filters, marker, and limit - images = [] - for image_id in self._ids(): - try: - image = self.show(context, image_id) - images.append(image) - except exception.NotFound: - continue - return images - - def show(self, context, image_id): - try: - with open(self._path_to(image_id)) as metadata_file: - image_meta = json.load(metadata_file) - if not self._is_image_available(context, image_meta): - raise exception.ImageNotFound(image_id=image_id) - return image_meta - except (IOError, ValueError): - raise exception.ImageNotFound(image_id=image_id) - - def show_by_name(self, context, name): - """Returns a dict containing image data for the given name.""" - # NOTE(vish): Not very efficient, but the local image service - # is for testing so it should be fine. - images = self.detail(context) - image = None - for cantidate in images: - if name == cantidate.get('name'): - image = cantidate - break - if image is None: - raise exception.ImageNotFound(image_id=name) - return image - - def get(self, context, image_id, data): - """Get image and metadata.""" - try: - with open(self._path_to(image_id)) as metadata_file: - metadata = json.load(metadata_file) - with open(self._path_to(image_id, 'image')) as image_file: - shutil.copyfileobj(image_file, data) - except (IOError, ValueError): - raise exception.ImageNotFound(image_id=image_id) - return metadata - - def create(self, context, metadata, data=None): - """Store the image data and return the new image.""" - image_id = random.randint(0, 2 ** 31 - 1) - image_path = self._path_to(image_id, None) - if not os.path.exists(image_path): - os.mkdir(image_path) - return self._store(context, image_id, metadata, data) - - def update(self, context, image_id, metadata, data=None): - """Replace the contents of the given image with the new data.""" - # NOTE(vish): show is to check if image is available - self.show(context, image_id) - return self._store(context, image_id, metadata, data) - - def _store(self, context, image_id, metadata, data=None): - metadata['id'] = image_id - try: - if data: - location = self._path_to(image_id, 'image') - with open(location, 'w') as image_file: - shutil.copyfileobj(data, image_file) - # NOTE(vish): update metadata similarly to glance - metadata['status'] = 'active' - metadata['location'] = location - with open(self._path_to(image_id), 'w') as metadata_file: - json.dump(metadata, metadata_file) - except (IOError, ValueError): - raise exception.ImageNotFound(image_id=image_id) - return metadata - - def delete(self, context, image_id): - """Delete the given image. - - :raises: ImageNotFound if the image does not exist. - - """ - # NOTE(vish): show is to check if image is available - self.show(context, image_id) - try: - shutil.rmtree(self._path_to(image_id, None)) - except (IOError, ValueError): - raise exception.ImageNotFound(image_id=image_id) - - def delete_all(self): - """Clears out all images in local directory.""" - for image_id in self._ids(): - shutil.rmtree(self._path_to(image_id, None)) diff --git a/nova/scheduler/api.py b/nova/scheduler/api.py index 09e7c9140..1bb047e2e 100644 --- a/nova/scheduler/api.py +++ b/nova/scheduler/api.py @@ -24,6 +24,7 @@ from nova import exception from nova import flags from nova import log as logging from nova import rpc +from nova import utils from eventlet import greenpool @@ -106,12 +107,14 @@ def _wrap_method(function, self): def _process(func, zone): """Worker stub for green thread pool. Give the worker an authenticated nova client and zone info.""" - nova = novaclient.OpenStack(zone.username, zone.password, zone.api_url) + nova = novaclient.OpenStack(zone.username, zone.password, None, + zone.api_url) nova.authenticate() return func(nova, zone) -def call_zone_method(context, method, errors_to_ignore=None, *args, **kwargs): +def call_zone_method(context, method_name, errors_to_ignore=None, + novaclient_collection_name='zones', *args, **kwargs): """Returns a list of (zone, call_result) objects.""" if not isinstance(errors_to_ignore, (list, tuple)): # This will also handle the default None @@ -121,7 +124,7 @@ def call_zone_method(context, method, errors_to_ignore=None, *args, **kwargs): results = [] for zone in db.zone_get_all(context): try: - nova = novaclient.OpenStack(zone.username, zone.password, + nova = novaclient.OpenStack(zone.username, zone.password, None, zone.api_url) nova.authenticate() except novaclient.exceptions.BadRequest, e: @@ -131,18 +134,16 @@ def call_zone_method(context, method, errors_to_ignore=None, *args, **kwargs): #TODO (dabo) - add logic for failure counts per zone, # with escalation after a given number of failures. continue - zone_method = getattr(nova.zones, method) + novaclient_collection = getattr(nova, novaclient_collection_name) + collection_method = getattr(novaclient_collection, method_name) def _error_trap(*args, **kwargs): try: - return zone_method(*args, **kwargs) + return collection_method(*args, **kwargs) except Exception as e: if type(e) in errors_to_ignore: return None - # TODO (dabo) - want to be able to re-raise here. - # Returning a string now; raising was causing issues. - # raise e - return "ERROR", "%s" % e + raise res = pool.spawn(_error_trap, *args, **kwargs) results.append((zone, res)) @@ -201,38 +202,78 @@ class RedirectResult(exception.Error): class reroute_compute(object): - """Decorator used to indicate that the method should - delegate the call the child zones if the db query - can't find anything.""" + """ + reroute_compute is responsible for trying to lookup a resource in the + current zone and if it's not found there, delegating the call to the + child zones. + + Since reroute_compute will be making 'cross-zone' calls, the ID for the + object must come in as a UUID-- if we receive an integer ID, we bail. + + The steps involved are: + + 1. Validate that item_id is UUID like + + 2. Lookup item by UUID in the zone local database + + 3. If the item was found, then extract integer ID, and pass that to + the wrapped method. (This ensures that zone-local code can + continue to use integer IDs). + + 4. If the item was not found, we delgate the call to a child zone + using the UUID. + """ def __init__(self, method_name): self.method_name = method_name + def _route_to_child_zones(self, context, collection, item_uuid): + if not FLAGS.enable_zone_routing: + raise exception.InstanceNotFound(instance_id=item_uuid) + + zones = db.zone_get_all(context) + if not zones: + raise exception.InstanceNotFound(instance_id=item_uuid) + + # Ask the children to provide an answer ... + LOG.debug(_("Asking child zones ...")) + result = self._call_child_zones(zones, + wrap_novaclient_function(_issue_novaclient_command, + collection, self.method_name, item_uuid)) + # Scrub the results and raise another exception + # so the API layers can bail out gracefully ... + raise RedirectResult(self.unmarshall_result(result)) + def __call__(self, f): def wrapped_f(*args, **kwargs): - collection, context, item_id = \ + collection, context, item_id_or_uuid = \ self.get_collection_context_and_id(args, kwargs) - try: - # Call the original function ... + + attempt_reroute = False + if utils.is_uuid_like(item_id_or_uuid): + item_uuid = item_id_or_uuid + try: + instance = db.instance_get_by_uuid(context, item_uuid) + except exception.InstanceNotFound, e: + # NOTE(sirp): since a UUID was passed in, we can attempt + # to reroute to a child zone + attempt_reroute = True + LOG.debug(_("Instance %(item_uuid)s not found " + "locally: '%(e)s'" % locals())) + else: + # NOTE(sirp): since we're not re-routing in this case, and + # we we were passed a UUID, we need to replace that UUID + # with an integer ID in the argument list so that the + # zone-local code can continue to use integer IDs. + item_id = instance['id'] + args = list(args) # needs to be mutable to replace + self.replace_uuid_with_id(args, kwargs, item_id) + + if attempt_reroute: + return self._route_to_child_zones(context, collection, + item_uuid) + else: return f(*args, **kwargs) - except exception.InstanceNotFound, e: - LOG.debug(_("Instance %(item_id)s not found " - "locally: '%(e)s'" % locals())) - - if not FLAGS.enable_zone_routing: - raise - - zones = db.zone_get_all(context) - if not zones: - raise - - # Ask the children to provide an answer ... - LOG.debug(_("Asking child zones ...")) - result = self._call_child_zones(zones, - wrap_novaclient_function(_issue_novaclient_command, - collection, self.method_name, item_id)) - # Scrub the results and raise another exception - # so the API layers can bail out gracefully ... - raise RedirectResult(self.unmarshall_result(result)) + return wrapped_f def _call_child_zones(self, zones, function): @@ -251,6 +292,18 @@ class reroute_compute(object): instance_id = args[2] return ("servers", context, instance_id) + @staticmethod + def replace_uuid_with_id(args, kwargs, replacement_id): + """ + Extracts the UUID parameter from the arg or kwarg list and replaces + it with an integer ID. + """ + if 'instance_id' in kwargs: + kwargs['instance_id'] = replacement_id + elif len(args) > 1: + args.pop(2) + args.insert(2, replacement_id) + def unmarshall_result(self, zone_responses): """Result is a list of responses from each child zone. Each decorator derivation is responsible to turning this diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index a29703aaf..6cb75aa8d 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -89,8 +89,8 @@ class SchedulerManager(manager.Manager): host = getattr(self.driver, driver_method)(elevated, *args, **kwargs) except AttributeError, e: - LOG.exception(_("Driver Method %(driver_method)s missing: %(e)s") - % locals()) + LOG.warning(_("Driver Method %(driver_method)s missing: %(e)s." + "Reverting to schedule()") % locals()) host = self.driver.schedule(elevated, topic, *args, **kwargs) if not host: diff --git a/nova/scheduler/simple.py b/nova/scheduler/simple.py index 87cdef11d..fc1b3142a 100644 --- a/nova/scheduler/simple.py +++ b/nova/scheduler/simple.py @@ -39,7 +39,7 @@ flags.DEFINE_integer("max_networks", 1000, class SimpleScheduler(chance.ChanceScheduler): """Implements Naive Scheduler that tries to find least loaded host.""" - def schedule_run_instance(self, context, instance_id, *_args, **_kwargs): + def _schedule_instance(self, context, instance_id, *_args, **_kwargs): """Picks a host that is up and has the fewest running instances.""" instance_ref = db.instance_get(context, instance_id) if (instance_ref['availability_zone'] @@ -75,6 +75,12 @@ class SimpleScheduler(chance.ChanceScheduler): " for this request. Is the appropriate" " service running?")) + def schedule_run_instance(self, context, instance_id, *_args, **_kwargs): + return self._schedule_instance(context, instance_id, *_args, **_kwargs) + + def schedule_start_instance(self, context, instance_id, *_args, **_kwargs): + return self._schedule_instance(context, instance_id, *_args, **_kwargs) + def schedule_create_volume(self, context, volume_id, *_args, **_kwargs): """Picks a host that is up and has the fewest volumes.""" volume_ref = db.volume_get(context, volume_id) diff --git a/nova/scheduler/zone_aware_scheduler.py b/nova/scheduler/zone_aware_scheduler.py index faa969124..e7bff2faa 100644 --- a/nova/scheduler/zone_aware_scheduler.py +++ b/nova/scheduler/zone_aware_scheduler.py @@ -88,9 +88,10 @@ class ZoneAwareScheduler(driver.Scheduler): instance_properties = request_spec['instance_properties'] name = instance_properties['display_name'] - image_id = instance_properties['image_id'] + image_ref = instance_properties['image_ref'] meta = instance_properties['metadata'] flavor_id = instance_type['flavorid'] + reservation_id = instance_properties['reservation_id'] files = kwargs['injected_files'] ipgroup = None # Not supported in OS API ... yet @@ -99,18 +100,20 @@ class ZoneAwareScheduler(driver.Scheduler): child_blob = zone_info['child_blob'] zone = db.zone_get(context, child_zone) url = zone.api_url - LOG.debug(_("Forwarding instance create call to child zone %(url)s") + LOG.debug(_("Forwarding instance create call to child zone %(url)s" + ". ReservationID=%(reservation_id)s") % locals()) nova = None try: - nova = novaclient.OpenStack(zone.username, zone.password, url) + nova = novaclient.OpenStack(zone.username, zone.password, None, + url) nova.authenticate() except novaclient.exceptions.BadRequest, e: raise exception.NotAuthorized(_("Bad credentials attempting " "to talk to zone at %(url)s.") % locals()) - nova.servers.create(name, image_id, flavor_id, ipgroup, meta, files, - child_blob) + nova.servers.create(name, image_ref, flavor_id, ipgroup, meta, files, + child_blob, reservation_id=reservation_id) def _provision_resource_from_blob(self, context, item, instance_id, request_spec, kwargs): @@ -182,7 +185,11 @@ class ZoneAwareScheduler(driver.Scheduler): if not build_plan: raise driver.NoValidHost(_('No hosts were available')) - for item in build_plan: + for num in xrange(request_spec['num_instances']): + if not build_plan: + break + + item = build_plan.pop(0) self._provision_resource(context, item, instance_id, request_spec, kwargs) diff --git a/nova/scheduler/zone_manager.py b/nova/scheduler/zone_manager.py index 3f483adff..ba7403c15 100644 --- a/nova/scheduler/zone_manager.py +++ b/nova/scheduler/zone_manager.py @@ -89,7 +89,8 @@ class ZoneState(object): def _call_novaclient(zone): """Call novaclient. Broken out for testing purposes.""" - client = novaclient.OpenStack(zone.username, zone.password, zone.api_url) + client = novaclient.OpenStack(zone.username, zone.password, None, + zone.api_url) return client.zones.info()._info diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index a10fb7433..f8d158ddd 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -39,7 +39,6 @@ from nova.api.openstack import limits from nova.auth.manager import User, Project import nova.image.fake from nova.image import glance -from nova.image import local from nova.image import service from nova.tests import fake_flags from nova.wsgi import Router diff --git a/nova/tests/api/openstack/test_api.py b/nova/tests/api/openstack/test_api.py index c63431a45..7321c329f 100644 --- a/nova/tests/api/openstack/test_api.py +++ b/nova/tests/api/openstack/test_api.py @@ -15,6 +15,8 @@ # License for the specific language governing permissions and limitations # under the License. +import json + import webob.exc import webob.dec @@ -23,6 +25,7 @@ from webob import Request from nova import test from nova.api import openstack from nova.api.openstack import faults +from nova.tests.api.openstack import fakes class APITest(test.TestCase): @@ -31,6 +34,24 @@ class APITest(test.TestCase): # simpler version of the app than fakes.wsgi_app return openstack.FaultWrapper(inner_app) + def test_malformed_json(self): + req = webob.Request.blank('/') + req.method = 'POST' + req.body = '{' + req.headers["content-type"] = "application/json" + + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 400) + + def test_malformed_xml(self): + req = webob.Request.blank('/') + req.method = 'POST' + req.body = '<hi im not xml>' + req.headers["content-type"] = "application/xml" + + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(res.status_int, 400) + def test_exceptions_are_converted_to_faults(self): @webob.dec.wsgify diff --git a/nova/tests/api/openstack/test_extensions.py b/nova/tests/api/openstack/test_extensions.py index 60914c0a3..697c62e5c 100644 --- a/nova/tests/api/openstack/test_extensions.py +++ b/nova/tests/api/openstack/test_extensions.py @@ -128,6 +128,11 @@ class ResourceExtensionTest(unittest.TestCase): self.assertEqual(response_body, response.body) +class InvalidExtension(object): + def get_alias(self): + return "THIRD" + + class ExtensionManagerTest(unittest.TestCase): response_body = "Try to say this Mr. Knox, sir..." @@ -144,6 +149,14 @@ class ExtensionManagerTest(unittest.TestCase): self.assertEqual(200, response.status_int) self.assertEqual(response_body, response.body) + def test_invalid_extensions(self): + app = openstack.APIRouterV11() + ext_midware = extensions.ExtensionMiddleware(app) + ext_mgr = ext_midware.ext_mgr + ext_mgr.add_extension(InvalidExtension()) + self.assertTrue('FOXNSOX' in ext_mgr.extensions) + self.assertTrue('THIRD' not in ext_mgr.extensions) + class ActionExtensionTest(unittest.TestCase): diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index be777df9b..e4204809f 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -135,36 +135,6 @@ class _BaseImageServiceTests(test.TestCase): return fixture -class LocalImageServiceTest(_BaseImageServiceTests): - - """Tests the local image service""" - - def setUp(self): - super(LocalImageServiceTest, self).setUp() - self.tempdir = tempfile.mkdtemp() - self.flags(images_path=self.tempdir) - self.stubs = stubout.StubOutForTesting() - service_class = 'nova.image.local.LocalImageService' - self.service = utils.import_object(service_class) - self.context = context.RequestContext(None, None) - - def tearDown(self): - shutil.rmtree(self.tempdir) - self.stubs.UnsetAll() - super(LocalImageServiceTest, self).tearDown() - - def test_get_all_ids_with_incorrect_directory_formats(self): - # create some old-style image directories (starting with 'ami-') - for x in [1, 2, 3]: - tempfile.mkstemp(prefix='ami-', dir=self.tempdir) - # create some valid image directories names - for x in ["1485baed", "1a60f0ee", "3123a73d"]: - os.makedirs(os.path.join(self.tempdir, x)) - found_image_ids = self.service._ids() - self.assertEqual(True, isinstance(found_image_ids, list)) - self.assertEqual(3, len(found_image_ids), len(found_image_ids)) - - class GlanceImageServiceTest(_BaseImageServiceTests): """Tests the Glance image service, in particular that metadata translation diff --git a/nova/tests/api/openstack/test_limits.py b/nova/tests/api/openstack/test_limits.py index 01613d1d8..38c959fae 100644 --- a/nova/tests/api/openstack/test_limits.py +++ b/nova/tests/api/openstack/test_limits.py @@ -672,8 +672,7 @@ class WsgiLimiterTest(BaseLimitTestSuite): """Only POSTs should work.""" requests = [] for method in ["GET", "PUT", "DELETE", "HEAD", "OPTIONS"]: - request = webob.Request.blank("/") - request.body = self._request_data("GET", "/something") + request = webob.Request.blank("/", method=method) response = request.get_response(self.app) self.assertEqual(response.status_int, 405) diff --git a/nova/tests/api/openstack/test_server_metadata.py b/nova/tests/api/openstack/test_server_metadata.py index b583d40fe..0431e68d2 100644 --- a/nova/tests/api/openstack/test_server_metadata.py +++ b/nova/tests/api/openstack/test_server_metadata.py @@ -21,6 +21,7 @@ import unittest import webob +from nova import exception from nova import flags from nova.api import openstack from nova.tests.api.openstack import fakes @@ -67,6 +68,14 @@ def stub_max_server_metadata(): return metadata +def return_server(context, server_id): + return {'id': server_id} + + +def return_server_nonexistant(context, server_id): + raise exception.InstanceNotFound() + + class ServerMetaDataTest(unittest.TestCase): def setUp(self): @@ -76,6 +85,7 @@ class ServerMetaDataTest(unittest.TestCase): fakes.FakeAuthDatabase.data = {} fakes.stub_out_auth(self.stubs) fakes.stub_out_key_pair_funcs(self.stubs) + self.stubs.Set(nova.db.api, 'instance_get', return_server) def tearDown(self): self.stubs.UnsetAll() @@ -92,6 +102,13 @@ class ServerMetaDataTest(unittest.TestCase): self.assertEqual('application/json', res.headers['Content-Type']) self.assertEqual('value1', res_dict['metadata']['key1']) + def test_index_nonexistant_server(self): + self.stubs.Set(nova.db.api, 'instance_get', return_server_nonexistant) + req = webob.Request.blank('/v1.1/servers/1/meta') + req.environ['api.version'] = '1.1' + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(404, res.status_int) + def test_index_no_data(self): self.stubs.Set(nova.db.api, 'instance_metadata_get', return_empty_server_metadata) @@ -114,13 +131,19 @@ class ServerMetaDataTest(unittest.TestCase): self.assertEqual('application/json', res.headers['Content-Type']) self.assertEqual('value5', res_dict['key5']) + def test_show_nonexistant_server(self): + self.stubs.Set(nova.db.api, 'instance_get', return_server_nonexistant) + req = webob.Request.blank('/v1.1/servers/1/meta/key5') + req.environ['api.version'] = '1.1' + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(404, res.status_int) + def test_show_meta_not_found(self): self.stubs.Set(nova.db.api, 'instance_metadata_get', return_empty_server_metadata) req = webob.Request.blank('/v1.1/servers/1/meta/key6') req.environ['api.version'] = '1.1' res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(404, res.status_int) def test_delete(self): @@ -132,6 +155,14 @@ class ServerMetaDataTest(unittest.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(200, res.status_int) + def test_delete_nonexistant_server(self): + self.stubs.Set(nova.db.api, 'instance_get', return_server_nonexistant) + req = webob.Request.blank('/v1.1/servers/1/meta/key5') + req.environ['api.version'] = '1.1' + req.method = 'DELETE' + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(404, res.status_int) + def test_create(self): self.stubs.Set(nova.db.api, 'instance_metadata_update_or_create', return_create_instance_metadata) @@ -141,8 +172,8 @@ class ServerMetaDataTest(unittest.TestCase): req.body = '{"metadata": {"key1": "value1"}}' req.headers["content-type"] = "application/json" res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(200, res.status_int) + res_dict = json.loads(res.body) self.assertEqual('application/json', res.headers['Content-Type']) self.assertEqual('value1', res_dict['metadata']['key1']) @@ -156,6 +187,16 @@ class ServerMetaDataTest(unittest.TestCase): res = req.get_response(fakes.wsgi_app()) self.assertEqual(400, res.status_int) + def test_create_nonexistant_server(self): + self.stubs.Set(nova.db.api, 'instance_get', return_server_nonexistant) + req = webob.Request.blank('/v1.1/servers/100/meta') + req.environ['api.version'] = '1.1' + req.method = 'POST' + req.body = '{"metadata": {"key1": "value1"}}' + req.headers["content-type"] = "application/json" + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(404, res.status_int) + def test_update_item(self): self.stubs.Set(nova.db.api, 'instance_metadata_update_or_create', return_create_instance_metadata) @@ -170,6 +211,16 @@ class ServerMetaDataTest(unittest.TestCase): res_dict = json.loads(res.body) self.assertEqual('value1', res_dict['key1']) + def test_update_item_nonexistant_server(self): + self.stubs.Set(nova.db.api, 'instance_get', return_server_nonexistant) + req = webob.Request.blank('/v1.1/servers/asdf/100/key1') + req.environ['api.version'] = '1.1' + req.method = 'PUT' + req.body = '{"key1": "value1"}' + req.headers["content-type"] = "application/json" + res = req.get_response(fakes.wsgi_app()) + self.assertEqual(404, res.status_int) + def test_update_item_empty_body(self): self.stubs.Set(nova.db.api, 'instance_metadata_update_or_create', return_create_instance_metadata) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 28ad4a417..b53c6c9be 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -31,10 +31,12 @@ from nova import test from nova import utils import nova.api.openstack from nova.api.openstack import servers +from nova.api.openstack import create_instance_helper import nova.compute.api from nova.compute import instance_types from nova.compute import power_state import nova.db.api +import nova.scheduler.api from nova.db.sqlalchemy.models import Instance from nova.db.sqlalchemy.models import InstanceMetadata import nova.image.fake @@ -47,10 +49,22 @@ FLAGS = flags.FLAGS FLAGS.verbose = True -def return_server(context, id): +FAKE_UUID = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' + + +def fake_gen_uuid(): + return FAKE_UUID + + +def return_server_by_id(context, id): return stub_instance(id) +def return_server_by_uuid(context, uuid): + id = 1 + return stub_instance(id, uuid=uuid) + + def return_server_with_addresses(private, public): def _return_server(context, id): return stub_instance(id, private_address=private, @@ -68,6 +82,34 @@ def return_servers(context, user_id=1): return [stub_instance(i, user_id) for i in xrange(5)] +def return_servers_by_reservation(context, reservation_id=""): + return [stub_instance(i, reservation_id) for i in xrange(5)] + + +def return_servers_by_reservation_empty(context, reservation_id=""): + return [] + + +def return_servers_from_child_zones_empty(*args, **kwargs): + return [] + + +def return_servers_from_child_zones(*args, **kwargs): + class Server(object): + pass + + zones = [] + for zone in xrange(3): + servers = [] + for server_id in xrange(5): + server = Server() + server._info = stub_instance(server_id, reservation_id="child") + servers.append(server) + + zones.append(("Zone%d" % zone, servers)) + return zones + + def return_security_group(context, instance_id, security_group_id): pass @@ -81,7 +123,8 @@ def instance_address(context, instance_id): def stub_instance(id, user_id=1, private_address=None, public_addresses=None, - host=None, power_state=0): + host=None, power_state=0, reservation_id="", + uuid=FAKE_UUID): metadata = [] metadata.append(InstanceMetadata(key='seq', value=id)) @@ -93,8 +136,13 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None, if host is not None: host = str(host) + # ReservationID isn't sent back, hack it in there. + server_name = "server%s" % id + if reservation_id != "": + server_name = "reservation_%s" % (reservation_id, ) + instance = { - "id": id, + "id": int(id), "admin_pass": "", "user_id": user_id, "project_id": "", @@ -113,16 +161,17 @@ def stub_instance(id, user_id=1, private_address=None, public_addresses=None, "host": host, "instance_type": dict(inst_type), "user_data": "", - "reservation_id": "", + "reservation_id": reservation_id, "mac_address": "", "scheduled_at": utils.utcnow(), "launched_at": utils.utcnow(), "terminated_at": utils.utcnow(), "availability_zone": "", - "display_name": "server%s" % id, + "display_name": server_name, "display_description": "", "locked": False, - "metadata": metadata} + "metadata": metadata, + "uuid": uuid} instance["fixed_ip"] = { "address": private_address, @@ -161,8 +210,11 @@ class ServersTest(test.TestCase): fakes.stub_out_auth(self.stubs) fakes.stub_out_key_pair_funcs(self.stubs) fakes.stub_out_image_service(self.stubs) + self.stubs.Set(utils, 'gen_uuid', fake_gen_uuid) self.stubs.Set(nova.db.api, 'instance_get_all', return_servers) - self.stubs.Set(nova.db.api, 'instance_get', return_server) + self.stubs.Set(nova.db.api, 'instance_get', return_server_by_id) + self.stubs.Set(nova.db, 'instance_get_by_uuid', + return_server_by_uuid) self.stubs.Set(nova.db.api, 'instance_get_all_by_user', return_servers) self.stubs.Set(nova.db.api, 'instance_add_security_group', @@ -194,6 +246,36 @@ class ServersTest(test.TestCase): self.assertEqual(res_dict['server']['id'], 1) self.assertEqual(res_dict['server']['name'], 'server1') + def test_get_server_by_uuid(self): + """ + The steps involved with resolving a UUID are pretty complicated; + here's what's happening in this scenario: + + 1. Show is calling `routing_get` + + 2. `routing_get` is wrapped by `reroute_compute` which does the work + of resolving requests to child zones. + + 3. `reroute_compute` looks up the UUID by hitting the stub + (returns_server_by_uuid) + + 4. Since the stub return that the record exists, `reroute_compute` + considers the request to be 'zone local', so it replaces the UUID + in the argument list with an integer ID and then calls the inner + function ('get'). + + 5. The call to `get` hits the other stub 'returns_server_by_id` which + has the UUID set to FAKE_UUID + + So, counterintuitively, we call `get` twice on the `show` command. + """ + req = webob.Request.blank('/v1.0/servers/%s' % FAKE_UUID) + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + self.assertEqual(res_dict['server']['id'], 1) + self.assertEqual(res_dict['server']['uuid'], FAKE_UUID) + self.assertEqual(res_dict['server']['name'], 'server1') + def test_get_server_by_id_v1_1(self): req = webob.Request.blank('/v1.1/servers/1') res = req.get_response(fakes.wsgi_app()) @@ -364,6 +446,57 @@ class ServersTest(test.TestCase): self.assertEqual(s.get('imageId', None), None) i += 1 + def test_get_server_list_with_reservation_id(self): + self.stubs.Set(nova.db.api, 'instance_get_all_by_reservation', + return_servers_by_reservation) + self.stubs.Set(nova.scheduler.api, 'call_zone_method', + return_servers_from_child_zones) + req = webob.Request.blank('/v1.0/servers?reservation_id=foo') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + i = 0 + for s in res_dict['servers']: + if '_is_precooked' in s: + self.assertEqual(s.get('reservation_id'), 'child') + else: + self.assertEqual(s.get('name'), 'server%d' % i) + i += 1 + + def test_get_server_list_with_reservation_id_empty(self): + self.stubs.Set(nova.db.api, 'instance_get_all_by_reservation', + return_servers_by_reservation_empty) + self.stubs.Set(nova.scheduler.api, 'call_zone_method', + return_servers_from_child_zones_empty) + req = webob.Request.blank('/v1.0/servers/detail?reservation_id=foo') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + i = 0 + for s in res_dict['servers']: + if '_is_precooked' in s: + self.assertEqual(s.get('reservation_id'), 'child') + else: + self.assertEqual(s.get('name'), 'server%d' % i) + i += 1 + + def test_get_server_list_with_reservation_id_details(self): + self.stubs.Set(nova.db.api, 'instance_get_all_by_reservation', + return_servers_by_reservation) + self.stubs.Set(nova.scheduler.api, 'call_zone_method', + return_servers_from_child_zones) + req = webob.Request.blank('/v1.0/servers/detail?reservation_id=foo') + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + i = 0 + for s in res_dict['servers']: + if '_is_precooked' in s: + self.assertEqual(s.get('reservation_id'), 'child') + else: + self.assertEqual(s.get('name'), 'server%d' % i) + i += 1 + def test_get_server_list_v1_1(self): req = webob.Request.blank('/v1.1/servers') res = req.get_response(fakes.wsgi_app()) @@ -454,7 +587,8 @@ class ServersTest(test.TestCase): def _setup_for_create_instance(self): """Shared implementation for tests below that create instance""" def instance_create(context, inst): - return {'id': '1', 'display_name': 'server_test'} + return {'id': 1, 'display_name': 'server_test', + 'uuid': FAKE_UUID} def server_update(context, id, params): return instance_create(context, id) @@ -483,7 +617,8 @@ class ServersTest(test.TestCase): self.stubs.Set(nova.db.api, 'queue_get_for', queue_get_for) self.stubs.Set(nova.network.manager.VlanManager, 'allocate_fixed_ip', fake_method) - self.stubs.Set(nova.api.openstack.servers.Controller, + self.stubs.Set( + nova.api.openstack.create_instance_helper.CreateInstanceHelper, "_get_kernel_ramdisk_from_image", kernel_ramdisk_mapping) self.stubs.Set(nova.compute.api.API, "_find_host", find_host) @@ -507,11 +642,64 @@ class ServersTest(test.TestCase): self.assertEqual(1, server['id']) self.assertEqual(2, server['flavorId']) self.assertEqual(3, server['imageId']) + self.assertEqual(FAKE_UUID, server['uuid']) self.assertEqual(res.status_int, 200) def test_create_instance(self): self._test_create_instance_helper() + def test_create_instance_has_uuid(self): + """Tests at the db-layer instead of API layer since that's where the + UUID is generated + """ + ctxt = context.RequestContext(1, 1) + values = {} + instance = nova.db.api.instance_create(ctxt, values) + expected = FAKE_UUID + self.assertEqual(instance['uuid'], expected) + + def test_create_instance_via_zones(self): + """Server generated ReservationID""" + self._setup_for_create_instance() + FLAGS.allow_admin_api = True + + body = dict(server=dict( + name='server_test', imageId=3, flavorId=2, + metadata={'hello': 'world', 'open': 'stack'}, + personality={})) + req = webob.Request.blank('/v1.0/zones/boot') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + + res = req.get_response(fakes.wsgi_app()) + + reservation_id = json.loads(res.body)['reservation_id'] + self.assertEqual(res.status_int, 200) + self.assertNotEqual(reservation_id, "") + self.assertNotEqual(reservation_id, None) + self.assertTrue(len(reservation_id) > 1) + + def test_create_instance_via_zones_with_resid(self): + """User supplied ReservationID""" + self._setup_for_create_instance() + FLAGS.allow_admin_api = True + + body = dict(server=dict( + name='server_test', imageId=3, flavorId=2, + metadata={'hello': 'world', 'open': 'stack'}, + personality={}, reservation_id='myresid')) + req = webob.Request.blank('/v1.0/zones/boot') + req.method = 'POST' + req.body = json.dumps(body) + req.headers["content-type"] = "application/json" + + res = req.get_response(fakes.wsgi_app()) + + reservation_id = json.loads(res.body)['reservation_id'] + self.assertEqual(res.status_int, 200) + self.assertEqual(reservation_id, "myresid") + def test_create_instance_no_key_pair(self): fakes.stub_out_key_pair_funcs(self.stubs, have_key_pair=False) self._test_create_instance_helper() @@ -1314,7 +1502,7 @@ class ServersTest(test.TestCase): self.assertEqual(res.status_int, 400) def test_resized_server_has_correct_status(self): - req = self.webreq('/1', 'GET', dict(resize=dict(flavorId=3))) + req = self.webreq('/1', 'GET') def fake_migration_get(*args): return {} @@ -1401,7 +1589,7 @@ class ServersTest(test.TestCase): class TestServerCreateRequestXMLDeserializer(unittest.TestCase): def setUp(self): - self.deserializer = servers.ServerXMLDeserializer() + self.deserializer = create_instance_helper.ServerXMLDeserializer() def test_minimal_request(self): serial_request = """ @@ -1721,7 +1909,8 @@ class TestServerInstanceCreation(test.TestCase): self.injected_files = kwargs['injected_files'] else: self.injected_files = None - return [{'id': '1234', 'display_name': 'fakeinstance'}] + return [{'id': '1234', 'display_name': 'fakeinstance', + 'uuid': FAKE_UUID}] def set_admin_password(self, *args, **kwargs): pass @@ -1733,7 +1922,8 @@ class TestServerInstanceCreation(test.TestCase): compute_api = MockComputeAPI() self.stubs.Set(nova.compute, 'API', make_stub_method(compute_api)) - self.stubs.Set(nova.api.openstack.servers.Controller, + self.stubs.Set( + nova.api.openstack.create_instance_helper.CreateInstanceHelper, '_get_kernel_ramdisk_from_image', make_stub_method((1, 1))) return compute_api @@ -1989,6 +2179,6 @@ class TestGetKernelRamdiskFromImage(test.TestCase): @staticmethod def _get_k_r(image_meta): """Rebinding function to a shorter name for convenience""" - kernel_id, ramdisk_id = \ - servers.Controller._do_get_kernel_ramdisk_from_image(image_meta) + kernel_id, ramdisk_id = create_instance_helper.CreateInstanceHelper. \ + _do_get_kernel_ramdisk_from_image(image_meta) return kernel_id, ramdisk_id diff --git a/nova/tests/api/openstack/test_wsgi.py b/nova/tests/api/openstack/test_wsgi.py index ebbdc9409..73a26a087 100644 --- a/nova/tests/api/openstack/test_wsgi.py +++ b/nova/tests/api/openstack/test_wsgi.py @@ -10,13 +10,13 @@ from nova.api.openstack import wsgi class RequestTest(test.TestCase): def test_content_type_missing(self): - request = wsgi.Request.blank('/tests/123') + request = wsgi.Request.blank('/tests/123', method='POST') request.body = "<body />" self.assertRaises(exception.InvalidContentType, request.get_content_type) def test_content_type_unsupported(self): - request = wsgi.Request.blank('/tests/123') + request = wsgi.Request.blank('/tests/123', method='POST') request.headers["Content-Type"] = "text/html" request.body = "asdf<br />" self.assertRaises(exception.InvalidContentType, @@ -89,6 +89,12 @@ class DictSerializerTest(test.TestCase): serializer.default = lambda x: 'trousers' self.assertEqual(serializer.serialize({}, 'update'), 'trousers') + def test_dispatch_action_None(self): + serializer = wsgi.DictSerializer() + serializer.create = lambda x: 'pants' + serializer.default = lambda x: 'trousers' + self.assertEqual(serializer.serialize({}, None), 'trousers') + class XMLDictSerializerTest(test.TestCase): def test_xml(self): @@ -123,6 +129,12 @@ class TextDeserializerTest(test.TestCase): deserializer.default = lambda x: 'trousers' self.assertEqual(deserializer.deserialize({}, 'update'), 'trousers') + def test_dispatch_action_None(self): + deserializer = wsgi.TextDeserializer() + deserializer.create = lambda x: 'pants' + deserializer.default = lambda x: 'trousers' + self.assertEqual(deserializer.deserialize({}, None), 'trousers') + class JSONDeserializerTest(test.TestCase): def test_json(self): @@ -171,11 +183,11 @@ class XMLDeserializerTest(test.TestCase): class ResponseSerializerTest(test.TestCase): def setUp(self): class JSONSerializer(object): - def serialize(self, data): + def serialize(self, data, action='default'): return 'pew_json' class XMLSerializer(object): - def serialize(self, data): + def serialize(self, data, action='default'): return 'pew_xml' self.serializers = { @@ -211,11 +223,11 @@ class ResponseSerializerTest(test.TestCase): class RequestDeserializerTest(test.TestCase): def setUp(self): class JSONDeserializer(object): - def deserialize(self, data): + def deserialize(self, data, action='default'): return 'pew_json' class XMLDeserializer(object): - def deserialize(self, data): + def deserialize(self, data, action='default'): return 'pew_xml' self.deserializers = { diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index ecefc464a..2297d2f0e 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -32,7 +32,7 @@ flags.DECLARE('fake_network', 'nova.network.manager') FLAGS['network_size'].SetDefault(8) FLAGS['num_networks'].SetDefault(2) FLAGS['fake_network'].SetDefault(True) -FLAGS['image_service'].SetDefault('nova.image.local.LocalImageService') +FLAGS['image_service'].SetDefault('nova.image.fake.FakeImageService') flags.DECLARE('num_shelves', 'nova.volume.driver') flags.DECLARE('blades_per_shelf', 'nova.volume.driver') flags.DECLARE('iscsi_num_targets', 'nova.volume.driver') diff --git a/nova/tests/integrated/api/client.py b/nova/tests/integrated/api/client.py index eb9a3056e..76c03c5fa 100644 --- a/nova/tests/integrated/api/client.py +++ b/nova/tests/integrated/api/client.py @@ -221,30 +221,30 @@ class TestOpenStackClient(object): return self.api_delete('/flavors/%s' % flavor_id) def get_volume(self, volume_id): - return self.api_get('/volumes/%s' % volume_id)['volume'] + return self.api_get('/os-volumes/%s' % volume_id)['volume'] def get_volumes(self, detail=True): - rel_url = '/volumes/detail' if detail else '/volumes' + rel_url = '/os-volumes/detail' if detail else '/os-volumes' return self.api_get(rel_url)['volumes'] def post_volume(self, volume): - return self.api_post('/volumes', volume)['volume'] + return self.api_post('/os-volumes', volume)['volume'] def delete_volume(self, volume_id): - return self.api_delete('/volumes/%s' % volume_id) + return self.api_delete('/os-volumes/%s' % volume_id) def get_server_volume(self, server_id, attachment_id): - return self.api_get('/servers/%s/volume_attachments/%s' % + return self.api_get('/servers/%s/os-volume_attachments/%s' % (server_id, attachment_id))['volumeAttachment'] def get_server_volumes(self, server_id): - return self.api_get('/servers/%s/volume_attachments' % + return self.api_get('/servers/%s/os-volume_attachments' % (server_id))['volumeAttachments'] def post_server_volume(self, server_id, volume_attachment): - return self.api_post('/servers/%s/volume_attachments' % + return self.api_post('/servers/%s/os-volume_attachments' % (server_id), volume_attachment)['volumeAttachment'] def delete_server_volume(self, server_id, attachment_id): - return self.api_delete('/servers/%s/volume_attachments/%s' % + return self.api_delete('/servers/%s/os-volume_attachments/%s' % (server_id, attachment_id)) diff --git a/nova/tests/scheduler/test_host_filter.py b/nova/tests/scheduler/test_host_filter.py index 07817cc5a..10eafde08 100644 --- a/nova/tests/scheduler/test_host_filter.py +++ b/nova/tests/scheduler/test_host_filter.py @@ -133,11 +133,11 @@ class HostFilterTestCase(test.TestCase): raw = ['or', ['and', ['<', '$compute.host_memory_free', 30], - ['<', '$compute.disk_available', 300] + ['<', '$compute.disk_available', 300], ], ['and', ['>', '$compute.host_memory_free', 70], - ['>', '$compute.disk_available', 700] + ['>', '$compute.disk_available', 700], ] ] cooked = json.dumps(raw) @@ -183,12 +183,12 @@ class HostFilterTestCase(test.TestCase): self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps([]))) self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps({}))) self.assertTrue(hf.filter_hosts(self.zone_manager, json.dumps( - ['not', True, False, True, False] + ['not', True, False, True, False], ))) try: hf.filter_hosts(self.zone_manager, json.dumps( - 'not', True, False, True, False + 'not', True, False, True, False, )) self.fail("Should give KeyError") except KeyError, e: diff --git a/nova/tests/scheduler/test_least_cost_scheduler.py b/nova/tests/scheduler/test_least_cost_scheduler.py index 506fa62fb..9a5318aee 100644 --- a/nova/tests/scheduler/test_least_cost_scheduler.py +++ b/nova/tests/scheduler/test_least_cost_scheduler.py @@ -44,7 +44,7 @@ class WeightedSumTestCase(test.TestCase): hosts = [ FakeHost(1, 512 * MB, 100), FakeHost(2, 256 * MB, 400), - FakeHost(3, 512 * MB, 100) + FakeHost(3, 512 * MB, 100), ] weighted_fns = [ @@ -96,7 +96,7 @@ class LeastCostSchedulerTestCase(test.TestCase): def test_noop_cost_fn(self): FLAGS.least_cost_scheduler_cost_functions = [ - 'nova.scheduler.least_cost.noop_cost_fn' + 'nova.scheduler.least_cost.noop_cost_fn', ] FLAGS.noop_cost_fn_weight = 1 @@ -110,7 +110,7 @@ class LeastCostSchedulerTestCase(test.TestCase): def test_cost_fn_weights(self): FLAGS.least_cost_scheduler_cost_functions = [ - 'nova.scheduler.least_cost.noop_cost_fn' + 'nova.scheduler.least_cost.noop_cost_fn', ] FLAGS.noop_cost_fn_weight = 2 @@ -124,7 +124,7 @@ class LeastCostSchedulerTestCase(test.TestCase): def test_fill_first_cost_fn(self): FLAGS.least_cost_scheduler_cost_functions = [ - 'nova.scheduler.least_cost.fill_first_cost_fn' + 'nova.scheduler.least_cost.fill_first_cost_fn', ] FLAGS.fill_first_cost_fn_weight = 1 diff --git a/nova/tests/scheduler/test_scheduler.py b/nova/tests/scheduler/test_scheduler.py index 50b6b52c6..4be59d411 100644 --- a/nova/tests/scheduler/test_scheduler.py +++ b/nova/tests/scheduler/test_scheduler.py @@ -48,6 +48,10 @@ flags.DECLARE('stub_network', 'nova.compute.manager') flags.DECLARE('instances_path', 'nova.compute.manager') +FAKE_UUID_NOT_FOUND = 'ffffffff-ffff-ffff-ffff-ffffffffffff' +FAKE_UUID = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' + + class TestDriver(driver.Scheduler): """Scheduler Driver for Tests""" def schedule(context, topic, *args, **kwargs): @@ -926,12 +930,23 @@ def zone_get_all(context): ] +def fake_instance_get_by_uuid(context, uuid): + if FAKE_UUID_NOT_FOUND: + raise exception.InstanceNotFound(instance_id=uuid) + else: + return {'id': 1} + + class FakeRerouteCompute(api.reroute_compute): + def __init__(self, method_name, id_to_return=1): + super(FakeRerouteCompute, self).__init__(method_name) + self.id_to_return = id_to_return + def _call_child_zones(self, zones, function): return [] def get_collection_context_and_id(self, args, kwargs): - return ("servers", None, 1) + return ("servers", None, self.id_to_return) def unmarshall_result(self, zone_responses): return dict(magic="found me") @@ -960,6 +975,8 @@ class ZoneRedirectTest(test.TestCase): self.stubs = stubout.StubOutForTesting() self.stubs.Set(db, 'zone_get_all', zone_get_all) + self.stubs.Set(db, 'instance_get_by_uuid', + fake_instance_get_by_uuid) self.enable_zone_routing = FLAGS.enable_zone_routing FLAGS.enable_zone_routing = True @@ -976,8 +993,19 @@ class ZoneRedirectTest(test.TestCase): except api.RedirectResult, e: self.fail(_("Successful database hit should succeed")) - def test_trap_not_found_locally(self): + def test_trap_not_found_locally_id_passed(self): + """When an integer ID is not found locally, we cannot reroute to + another zone, so just return InstanceNotFound exception + """ decorator = FakeRerouteCompute("foo") + self.assertRaises(exception.InstanceNotFound, + decorator(go_boom), None, None, 1) + + def test_trap_not_found_locally_uuid_passed(self): + """When a UUID is found, if the item isn't found locally, we should + try to reroute to a child zone to see if they have it + """ + decorator = FakeRerouteCompute("foo", id_to_return=FAKE_UUID_NOT_FOUND) try: result = decorator(go_boom)(None, None, 1) self.assertFail(_("Should have rerouted.")) @@ -1110,10 +1138,4 @@ class CallZoneMethodTest(test.TestCase): def test_call_zone_method_generates_exception(self): context = {} method = 'raises_exception' - results = api.call_zone_method(context, method) - - # FIXME(sirp): for now the _error_trap code is catching errors and - # converting them to a ("ERROR", "string") tuples. The code (and this - # test) should eventually handle real exceptions. - expected = [(1, ('ERROR', 'testing'))] - self.assertEqual(expected, results) + self.assertRaises(Exception, api.call_zone_method, context, method) diff --git a/nova/tests/scheduler/test_zone_aware_scheduler.py b/nova/tests/scheduler/test_zone_aware_scheduler.py index 9f70b9dbc..37c6488cc 100644 --- a/nova/tests/scheduler/test_zone_aware_scheduler.py +++ b/nova/tests/scheduler/test_zone_aware_scheduler.py @@ -201,7 +201,7 @@ class ZoneAwareSchedulerTestCase(test.TestCase): 'instance_properties': {}, 'instance_type': {}, 'filter_driver': 'nova.scheduler.host_filter.AllHostsFilter', - 'blob': "Non-None blob data" + 'blob': "Non-None blob data", } result = sched.schedule_run_instance(None, 1, request_spec) diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index 7c0331eff..20b20fcbf 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -89,7 +89,7 @@ class FakeHttplibConnection(object): class XmlConversionTestCase(test.TestCase): """Unit test api xml conversion""" def test_number_conversion(self): - conv = apirequest._try_convert + conv = ec2utils._try_convert self.assertEqual(conv('None'), None) self.assertEqual(conv('True'), True) self.assertEqual(conv('False'), False) diff --git a/nova/tests/test_cloud.py b/nova/tests/test_cloud.py index 13046f861..6327734f5 100644 --- a/nova/tests/test_cloud.py +++ b/nova/tests/test_cloud.py @@ -35,7 +35,7 @@ from nova import utils from nova.auth import manager from nova.api.ec2 import cloud from nova.api.ec2 import ec2utils -from nova.image import local +from nova.image import fake FLAGS = flags.FLAGS @@ -56,6 +56,7 @@ class CloudTestCase(test.TestCase): self.compute = self.start_service('compute') self.scheduter = self.start_service('scheduler') self.network = self.start_service('network') + self.volume = self.start_service('volume') self.image_service = utils.import_object(FLAGS.image_service) self.manager = manager.AuthManager() @@ -69,8 +70,8 @@ class CloudTestCase(test.TestCase): return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, 'type': 'machine', 'image_state': 'available'}} - self.stubs.Set(local.LocalImageService, 'show', fake_show) - self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show) + self.stubs.Set(fake._FakeImageService, 'show', fake_show) + self.stubs.Set(fake._FakeImageService, 'show_by_name', fake_show) # NOTE(vish): set up a manual wait so rpc.cast has a chance to finish rpc_cast = rpc.cast @@ -303,7 +304,7 @@ class CloudTestCase(test.TestCase): def fake_show_none(meh, context, id): raise exception.ImageNotFound(image_id='bad_image_id') - self.stubs.Set(local.LocalImageService, 'detail', fake_detail) + self.stubs.Set(fake._FakeImageService, 'detail', fake_detail) # list all result1 = describe_images(self.context) result1 = result1['imagesSet'][0] @@ -317,8 +318,8 @@ class CloudTestCase(test.TestCase): self.assertEqual(2, len(result3['imagesSet'])) # provide an non-existing image_id self.stubs.UnsetAll() - self.stubs.Set(local.LocalImageService, 'show', fake_show_none) - self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show_none) + self.stubs.Set(fake._FakeImageService, 'show', fake_show_none) + self.stubs.Set(fake._FakeImageService, 'show_by_name', fake_show_none) self.assertRaises(exception.ImageNotFound, describe_images, self.context, ['ami-fake']) @@ -329,8 +330,8 @@ class CloudTestCase(test.TestCase): return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, 'type': 'machine'}, 'is_public': True} - self.stubs.Set(local.LocalImageService, 'show', fake_show) - self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show) + self.stubs.Set(fake._FakeImageService, 'show', fake_show) + self.stubs.Set(fake._FakeImageService, 'show_by_name', fake_show) result = describe_image_attribute(self.context, 'ami-00000001', 'launchPermission') self.assertEqual([{'group': 'all'}], result['launchPermission']) @@ -345,9 +346,9 @@ class CloudTestCase(test.TestCase): def fake_update(meh, context, image_id, metadata, data=None): return metadata - self.stubs.Set(local.LocalImageService, 'show', fake_show) - self.stubs.Set(local.LocalImageService, 'show_by_name', fake_show) - self.stubs.Set(local.LocalImageService, 'update', fake_update) + self.stubs.Set(fake._FakeImageService, 'show', fake_show) + self.stubs.Set(fake._FakeImageService, 'show_by_name', fake_show) + self.stubs.Set(fake._FakeImageService, 'update', fake_update) result = modify_image_attribute(self.context, 'ami-00000001', 'launchPermission', 'add', user_group=['all']) @@ -359,7 +360,7 @@ class CloudTestCase(test.TestCase): def fake_delete(self, context, id): return None - self.stubs.Set(local.LocalImageService, 'delete', fake_delete) + self.stubs.Set(fake._FakeImageService, 'delete', fake_delete) # valid image result = deregister_image(self.context, 'ami-00000001') self.assertEqual(result['imageId'], 'ami-00000001') @@ -369,18 +370,25 @@ class CloudTestCase(test.TestCase): def fake_detail_empty(self, context): return [] - self.stubs.Set(local.LocalImageService, 'detail', fake_detail_empty) + self.stubs.Set(fake._FakeImageService, 'detail', fake_detail_empty) self.assertRaises(exception.ImageNotFound, deregister_image, self.context, 'ami-bad001') - def test_console_output(self): - instance_type = FLAGS.default_instance_type - max_count = 1 - kwargs = {'image_id': 'ami-1', - 'instance_type': instance_type, - 'max_count': max_count} + def _run_instance(self, **kwargs): rv = self.cloud.run_instances(self.context, **kwargs) instance_id = rv['instancesSet'][0]['instanceId'] + return instance_id + + def _run_instance_wait(self, **kwargs): + ec2_instance_id = self._run_instance(**kwargs) + self._wait_for_running(ec2_instance_id) + return ec2_instance_id + + def test_console_output(self): + instance_id = self._run_instance( + image_id='ami-1', + instance_type=FLAGS.default_instance_type, + max_count=1) output = self.cloud.get_console_output(context=self.context, instance_id=[instance_id]) self.assertEquals(b64decode(output['output']), 'FAKE CONSOLE?OUTPUT') @@ -389,9 +397,7 @@ class CloudTestCase(test.TestCase): rv = self.cloud.terminate_instances(self.context, [instance_id]) def test_ajax_console(self): - kwargs = {'image_id': 'ami-1'} - rv = self.cloud.run_instances(self.context, **kwargs) - instance_id = rv['instancesSet'][0]['instanceId'] + instance_id = self._run_instance(image_id='ami-1') output = self.cloud.get_ajax_console(context=self.context, instance_id=[instance_id]) self.assertEquals(output['url'], @@ -457,6 +463,12 @@ class CloudTestCase(test.TestCase): self.cloud.delete_key_pair(self.context, 'test') def test_run_instances(self): + # stub out the rpc call + def stub_cast(*args, **kwargs): + pass + + self.stubs.Set(rpc, 'cast', stub_cast) + kwargs = {'image_id': FLAGS.default_image, 'instance_type': FLAGS.default_instance_type, 'max_count': 1} @@ -466,7 +478,7 @@ class CloudTestCase(test.TestCase): self.assertEqual(instance['imageId'], 'ami-00000001') self.assertEqual(instance['displayName'], 'Server 1') self.assertEqual(instance['instanceId'], 'i-00000001') - self.assertEqual(instance['instanceState']['name'], 'networking') + self.assertEqual(instance['instanceState']['name'], 'scheduling') self.assertEqual(instance['instanceType'], 'm1.small') def test_run_instances_image_state_none(self): @@ -480,7 +492,7 @@ class CloudTestCase(test.TestCase): 'type': 'machine'}} self.stubs.UnsetAll() - self.stubs.Set(local.LocalImageService, 'show', fake_show_no_state) + self.stubs.Set(fake._FakeImageService, 'show', fake_show_no_state) self.assertRaises(exception.ApiError, run_instances, self.context, **kwargs) @@ -495,7 +507,7 @@ class CloudTestCase(test.TestCase): 'type': 'machine', 'image_state': 'decrypting'}} self.stubs.UnsetAll() - self.stubs.Set(local.LocalImageService, 'show', fake_show_decrypt) + self.stubs.Set(fake._FakeImageService, 'show', fake_show_decrypt) self.assertRaises(exception.ApiError, run_instances, self.context, **kwargs) @@ -509,7 +521,7 @@ class CloudTestCase(test.TestCase): return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1, 'type': 'machine'}, 'status': 'active'} - self.stubs.Set(local.LocalImageService, 'show', fake_show_stat_active) + self.stubs.Set(fake._FakeImageService, 'show', fake_show_stat_active) result = run_instances(self.context, **kwargs) self.assertEqual(len(result['instancesSet']), 1) @@ -538,7 +550,9 @@ class CloudTestCase(test.TestCase): def test_update_of_instance_wont_update_private_fields(self): inst = db.instance_create(self.context, {}) - self.cloud.update_instance(self.context, inst['id'], + ec2_id = ec2utils.id_to_ec2_id(inst['id']) + self.cloud.update_instance(self.context, ec2_id, + display_name='c00l 1m4g3', mac_address='DE:AD:BE:EF') inst = db.instance_get(self.context, inst['id']) self.assertEqual(None, inst['mac_address']) @@ -561,3 +575,299 @@ class CloudTestCase(test.TestCase): vol = db.volume_get(self.context, vol['id']) self.assertEqual(None, vol['mountpoint']) db.volume_destroy(self.context, vol['id']) + + def _restart_compute_service(self, periodic_interval=None): + """restart compute service. NOTE: fake driver forgets all instances.""" + self.compute.kill() + if periodic_interval: + self.compute = self.start_service( + 'compute', periodic_interval=periodic_interval) + else: + self.compute = self.start_service('compute') + + def _wait_for_state(self, ctxt, instance_id, predicate): + """Wait for an stopping instance to be a given state""" + id = ec2utils.ec2_id_to_id(instance_id) + while True: + info = self.cloud.compute_api.get(context=ctxt, instance_id=id) + LOG.debug(info) + if predicate(info): + break + greenthread.sleep(1) + + def _wait_for_running(self, instance_id): + def is_running(info): + return info['state_description'] == 'running' + self._wait_for_state(self.context, instance_id, is_running) + + def _wait_for_stopped(self, instance_id): + def is_stopped(info): + return info['state_description'] == 'stopped' + self._wait_for_state(self.context, instance_id, is_stopped) + + def _wait_for_terminate(self, instance_id): + def is_deleted(info): + return info['deleted'] + elevated = self.context.elevated(read_deleted=True) + self._wait_for_state(elevated, instance_id, is_deleted) + + def test_stop_start_instance(self): + """Makes sure stop/start instance works""" + # enforce periodic tasks run in short time to avoid wait for 60s. + self._restart_compute_service(periodic_interval=0.3) + + kwargs = {'image_id': 'ami-1', + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1, } + instance_id = self._run_instance_wait(**kwargs) + + # a running instance can't be started. It is just ignored. + result = self.cloud.start_instances(self.context, [instance_id]) + greenthread.sleep(0.3) + self.assertTrue(result) + + result = self.cloud.stop_instances(self.context, [instance_id]) + greenthread.sleep(0.3) + self.assertTrue(result) + self._wait_for_stopped(instance_id) + + result = self.cloud.start_instances(self.context, [instance_id]) + greenthread.sleep(0.3) + self.assertTrue(result) + self._wait_for_running(instance_id) + + result = self.cloud.stop_instances(self.context, [instance_id]) + greenthread.sleep(0.3) + self.assertTrue(result) + self._wait_for_stopped(instance_id) + + result = self.cloud.terminate_instances(self.context, [instance_id]) + greenthread.sleep(0.3) + self.assertTrue(result) + + self._restart_compute_service() + + def _volume_create(self): + kwargs = {'status': 'available', + 'host': self.volume.host, + 'size': 1, + 'attach_status': 'detached', } + return db.volume_create(self.context, kwargs) + + def _assert_volume_attached(self, vol, instance_id, mountpoint): + self.assertEqual(vol['instance_id'], instance_id) + self.assertEqual(vol['mountpoint'], mountpoint) + self.assertEqual(vol['status'], "in-use") + self.assertEqual(vol['attach_status'], "attached") + + def _assert_volume_detached(self, vol): + self.assertEqual(vol['instance_id'], None) + self.assertEqual(vol['mountpoint'], None) + self.assertEqual(vol['status'], "available") + self.assertEqual(vol['attach_status'], "detached") + + def test_stop_start_with_volume(self): + """Make sure run instance with block device mapping works""" + + # enforce periodic tasks run in short time to avoid wait for 60s. + self._restart_compute_service(periodic_interval=0.3) + + vol1 = self._volume_create() + vol2 = self._volume_create() + kwargs = {'image_id': 'ami-1', + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1, + 'block_device_mapping': [{'device_name': '/dev/vdb', + 'volume_id': vol1['id'], + 'delete_on_termination': False, }, + {'device_name': '/dev/vdc', + 'volume_id': vol2['id'], + 'delete_on_termination': True, }, + ]} + ec2_instance_id = self._run_instance_wait(**kwargs) + instance_id = ec2utils.ec2_id_to_id(ec2_instance_id) + + vols = db.volume_get_all_by_instance(self.context, instance_id) + self.assertEqual(len(vols), 2) + for vol in vols: + self.assertTrue(vol['id'] == vol1['id'] or vol['id'] == vol2['id']) + + vol = db.volume_get(self.context, vol1['id']) + self._assert_volume_attached(vol, instance_id, '/dev/vdb') + + vol = db.volume_get(self.context, vol2['id']) + self._assert_volume_attached(vol, instance_id, '/dev/vdc') + + result = self.cloud.stop_instances(self.context, [ec2_instance_id]) + self.assertTrue(result) + self._wait_for_stopped(ec2_instance_id) + + vol = db.volume_get(self.context, vol1['id']) + self._assert_volume_detached(vol) + vol = db.volume_get(self.context, vol2['id']) + self._assert_volume_detached(vol) + + self.cloud.start_instances(self.context, [ec2_instance_id]) + self._wait_for_running(ec2_instance_id) + vols = db.volume_get_all_by_instance(self.context, instance_id) + self.assertEqual(len(vols), 2) + for vol in vols: + self.assertTrue(vol['id'] == vol1['id'] or vol['id'] == vol2['id']) + self.assertTrue(vol['mountpoint'] == '/dev/vdb' or + vol['mountpoint'] == '/dev/vdc') + self.assertEqual(vol['instance_id'], instance_id) + self.assertEqual(vol['status'], "in-use") + self.assertEqual(vol['attach_status'], "attached") + + self.cloud.terminate_instances(self.context, [ec2_instance_id]) + greenthread.sleep(0.3) + + admin_ctxt = context.get_admin_context(read_deleted=False) + vol = db.volume_get(admin_ctxt, vol1['id']) + self.assertFalse(vol['deleted']) + db.volume_destroy(self.context, vol1['id']) + + greenthread.sleep(0.3) + admin_ctxt = context.get_admin_context(read_deleted=True) + vol = db.volume_get(admin_ctxt, vol2['id']) + self.assertTrue(vol['deleted']) + + self._restart_compute_service() + + def test_stop_with_attached_volume(self): + """Make sure attach info is reflected to block device mapping""" + # enforce periodic tasks run in short time to avoid wait for 60s. + self._restart_compute_service(periodic_interval=0.3) + + vol1 = self._volume_create() + vol2 = self._volume_create() + kwargs = {'image_id': 'ami-1', + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1, + 'block_device_mapping': [{'device_name': '/dev/vdb', + 'volume_id': vol1['id'], + 'delete_on_termination': True}]} + ec2_instance_id = self._run_instance_wait(**kwargs) + instance_id = ec2utils.ec2_id_to_id(ec2_instance_id) + + vols = db.volume_get_all_by_instance(self.context, instance_id) + self.assertEqual(len(vols), 1) + for vol in vols: + self.assertEqual(vol['id'], vol1['id']) + self._assert_volume_attached(vol, instance_id, '/dev/vdb') + + vol = db.volume_get(self.context, vol2['id']) + self._assert_volume_detached(vol) + + self.cloud.compute_api.attach_volume(self.context, + instance_id=instance_id, + volume_id=vol2['id'], + device='/dev/vdc') + greenthread.sleep(0.3) + vol = db.volume_get(self.context, vol2['id']) + self._assert_volume_attached(vol, instance_id, '/dev/vdc') + + self.cloud.compute_api.detach_volume(self.context, + volume_id=vol1['id']) + greenthread.sleep(0.3) + vol = db.volume_get(self.context, vol1['id']) + self._assert_volume_detached(vol) + + result = self.cloud.stop_instances(self.context, [ec2_instance_id]) + self.assertTrue(result) + self._wait_for_stopped(ec2_instance_id) + + for vol_id in (vol1['id'], vol2['id']): + vol = db.volume_get(self.context, vol_id) + self._assert_volume_detached(vol) + + self.cloud.start_instances(self.context, [ec2_instance_id]) + self._wait_for_running(ec2_instance_id) + vols = db.volume_get_all_by_instance(self.context, instance_id) + self.assertEqual(len(vols), 1) + for vol in vols: + self.assertEqual(vol['id'], vol2['id']) + self._assert_volume_attached(vol, instance_id, '/dev/vdc') + + vol = db.volume_get(self.context, vol1['id']) + self._assert_volume_detached(vol) + + self.cloud.terminate_instances(self.context, [ec2_instance_id]) + greenthread.sleep(0.3) + + for vol_id in (vol1['id'], vol2['id']): + vol = db.volume_get(self.context, vol_id) + self.assertEqual(vol['id'], vol_id) + self._assert_volume_detached(vol) + db.volume_destroy(self.context, vol_id) + + self._restart_compute_service() + + def _create_snapshot(self, ec2_volume_id): + result = self.cloud.create_snapshot(self.context, + volume_id=ec2_volume_id) + greenthread.sleep(0.3) + return result['snapshotId'] + + def test_run_with_snapshot(self): + """Makes sure run/stop/start instance with snapshot works.""" + vol = self._volume_create() + ec2_volume_id = ec2utils.id_to_ec2_id(vol['id'], 'vol-%08x') + + ec2_snapshot1_id = self._create_snapshot(ec2_volume_id) + snapshot1_id = ec2utils.ec2_id_to_id(ec2_snapshot1_id) + ec2_snapshot2_id = self._create_snapshot(ec2_volume_id) + snapshot2_id = ec2utils.ec2_id_to_id(ec2_snapshot2_id) + + kwargs = {'image_id': 'ami-1', + 'instance_type': FLAGS.default_instance_type, + 'max_count': 1, + 'block_device_mapping': [{'device_name': '/dev/vdb', + 'snapshot_id': snapshot1_id, + 'delete_on_termination': False, }, + {'device_name': '/dev/vdc', + 'snapshot_id': snapshot2_id, + 'delete_on_termination': True}]} + ec2_instance_id = self._run_instance_wait(**kwargs) + instance_id = ec2utils.ec2_id_to_id(ec2_instance_id) + + vols = db.volume_get_all_by_instance(self.context, instance_id) + self.assertEqual(len(vols), 2) + vol1_id = None + vol2_id = None + for vol in vols: + snapshot_id = vol['snapshot_id'] + if snapshot_id == snapshot1_id: + vol1_id = vol['id'] + mountpoint = '/dev/vdb' + elif snapshot_id == snapshot2_id: + vol2_id = vol['id'] + mountpoint = '/dev/vdc' + else: + self.fail() + + self._assert_volume_attached(vol, instance_id, mountpoint) + + self.assertTrue(vol1_id) + self.assertTrue(vol2_id) + + self.cloud.terminate_instances(self.context, [ec2_instance_id]) + greenthread.sleep(0.3) + self._wait_for_terminate(ec2_instance_id) + + greenthread.sleep(0.3) + admin_ctxt = context.get_admin_context(read_deleted=False) + vol = db.volume_get(admin_ctxt, vol1_id) + self._assert_volume_detached(vol) + self.assertFalse(vol['deleted']) + db.volume_destroy(self.context, vol1_id) + + greenthread.sleep(0.3) + admin_ctxt = context.get_admin_context(read_deleted=True) + vol = db.volume_get(admin_ctxt, vol2_id) + self.assertTrue(vol['deleted']) + + for snapshot_id in (ec2_snapshot1_id, ec2_snapshot2_id): + self.cloud.delete_snapshot(self.context, snapshot_id) + greenthread.sleep(0.3) + db.volume_destroy(self.context, vol['id']) diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index b4ac2dbc4..439508b27 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -22,21 +22,21 @@ Tests For Compute import mox import stubout +from nova.auth import manager from nova import compute +from nova.compute import instance_types +from nova.compute import manager as compute_manager +from nova.compute import power_state from nova import context from nova import db +from nova.db.sqlalchemy import models from nova import exception from nova import flags +import nova.image.fake from nova import log as logging from nova import rpc from nova import test from nova import utils -from nova.auth import manager -from nova.compute import instance_types -from nova.compute import manager as compute_manager -from nova.compute import power_state -from nova.db.sqlalchemy import models -from nova.image import local LOG = logging.getLogger('nova.tests.compute') FLAGS = flags.FLAGS @@ -73,7 +73,7 @@ class ComputeTestCase(test.TestCase): def fake_show(meh, context, id): return {'id': 1, 'properties': {'kernel_id': 1, 'ramdisk_id': 1}} - self.stubs.Set(local.LocalImageService, 'show', fake_show) + self.stubs.Set(nova.image.fake._FakeImageService, 'show', fake_show) def tearDown(self): self.manager.delete_user(self.user) @@ -228,6 +228,21 @@ class ComputeTestCase(test.TestCase): self.assert_(instance_ref['launched_at'] < terminate) self.assert_(instance_ref['deleted_at'] > terminate) + def test_stop(self): + """Ensure instance can be stopped""" + instance_id = self._create_instance() + self.compute.run_instance(self.context, instance_id) + self.compute.stop_instance(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) + + def test_start(self): + """Ensure instance can be started""" + instance_id = self._create_instance() + self.compute.run_instance(self.context, instance_id) + self.compute.stop_instance(self.context, instance_id) + self.compute.start_instance(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) + def test_pause(self): """Ensure instance can be paused""" instance_id = self._create_instance() @@ -266,6 +281,14 @@ class ComputeTestCase(test.TestCase): "File Contents") self.compute.terminate_instance(self.context, instance_id) + def test_agent_update(self): + """Ensure instance can have its agent updated""" + instance_id = self._create_instance() + self.compute.run_instance(self.context, instance_id) + self.compute.agent_update(self.context, instance_id, + 'http://127.0.0.1/agent', '00112233445566778899aabbccddeeff') + self.compute.terminate_instance(self.context, instance_id) + def test_snapshot(self): """Ensure instance can be snapshotted""" instance_id = self._create_instance() diff --git a/nova/tests/test_crypto.py b/nova/tests/test_crypto.py index 945d78794..6c25b396e 100644 --- a/nova/tests/test_crypto.py +++ b/nova/tests/test_crypto.py @@ -16,7 +16,11 @@ Tests for Crypto module. """ +import mox +import stubout + from nova import crypto +from nova import db from nova import test @@ -46,3 +50,82 @@ class SymmetricKeyTestCase(test.TestCase): plain = decrypt(cipher_text) self.assertEquals(plain_text, plain) + + +class RevokeCertsTest(test.TestCase): + + def setUp(self): + super(RevokeCertsTest, self).setUp() + self.stubs = stubout.StubOutForTesting() + + def tearDown(self): + self.stubs.UnsetAll() + super(RevokeCertsTest, self).tearDown() + + def test_revoke_certs_by_user_and_project(self): + user_id = 'test_user' + project_id = 2 + file_name = 'test_file' + + def mock_certificate_get_all_by_user_and_project(context, + user_id, + project_id): + + return [{"user_id": user_id, "project_id": project_id, + "file_name": file_name}] + + self.stubs.Set(db, 'certificate_get_all_by_user_and_project', + mock_certificate_get_all_by_user_and_project) + + self.mox.StubOutWithMock(crypto, 'revoke_cert') + crypto.revoke_cert(project_id, file_name) + + self.mox.ReplayAll() + + crypto.revoke_certs_by_user_and_project(user_id, project_id) + + self.mox.VerifyAll() + + def test_revoke_certs_by_user(self): + user_id = 'test_user' + project_id = 2 + file_name = 'test_file' + + def mock_certificate_get_all_by_user(context, user_id): + + return [{"user_id": user_id, "project_id": project_id, + "file_name": file_name}] + + self.stubs.Set(db, 'certificate_get_all_by_user', + mock_certificate_get_all_by_user) + + self.mox.StubOutWithMock(crypto, 'revoke_cert') + crypto.revoke_cert(project_id, mox.IgnoreArg()) + + self.mox.ReplayAll() + + crypto.revoke_certs_by_user(user_id) + + self.mox.VerifyAll() + + def test_revoke_certs_by_project(self): + user_id = 'test_user' + project_id = 2 + file_name = 'test_file' + + def mock_certificate_get_all_by_project(context, project_id): + + return [{"user_id": user_id, "project_id": project_id, + "file_name": file_name}] + + self.stubs.Set(db, 'certificate_get_all_by_project', + mock_certificate_get_all_by_project) + + self.mox.StubOutWithMock(crypto, 'revoke_cert') + crypto.revoke_cert(project_id, mox.IgnoreArg()) + + self.mox.ReplayAll() + + crypto.revoke_certs_by_project(project_id) + + self.mox.VerifyAll() diff --git a/nova/tests/test_utils.py b/nova/tests/test_utils.py index 8f7e83c3e..3a3f914e4 100644 --- a/nova/tests/test_utils.py +++ b/nova/tests/test_utils.py @@ -275,3 +275,21 @@ class GenericUtilsTestCase(test.TestCase): # error case result = utils.parse_server_string('www.exa:mple.com:8443') self.assertEqual(('', ''), result) + + +class IsUUIDLikeTestCase(test.TestCase): + def assertUUIDLike(self, val, expected): + result = utils.is_uuid_like(val) + self.assertEqual(result, expected) + + def test_good_uuid(self): + val = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa' + self.assertUUIDLike(val, True) + + def test_integer_passed(self): + val = 1 + self.assertUUIDLike(val, False) + + def test_non_uuid_string_passed(self): + val = 'foo-fooo' + self.assertUUIDLike(val, False) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 3a175b106..d9a514745 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -33,12 +33,12 @@ from nova import utils from nova.auth import manager from nova.compute import instance_types from nova.compute import power_state +from nova import exception from nova.virt import xenapi_conn from nova.virt.xenapi import fake as xenapi_fake from nova.virt.xenapi import volume_utils +from nova.virt.xenapi import vmops from nova.virt.xenapi import vm_utils -from nova.virt.xenapi.vmops import SimpleDH -from nova.virt.xenapi.vmops import VMOps from nova.tests.db import fakes as db_fakes from nova.tests.xenapi import stubs from nova.tests.glance import stubs as glance_stubs @@ -84,7 +84,8 @@ class XenAPIVolumeTestCase(test.TestCase): 'ramdisk_id': 3, 'instance_type_id': '3', # m1.large 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': 'linux'} + 'os_type': 'linux', + 'architecture': 'x86-64'} def _create_volume(self, size='0'): """Create a volume object.""" @@ -191,7 +192,7 @@ class XenAPIVMTestCase(test.TestCase): stubs.stubout_get_this_vm_uuid(self.stubs) stubs.stubout_stream_disk(self.stubs) stubs.stubout_is_vdi_pv(self.stubs) - self.stubs.Set(VMOps, 'reset_network', reset_network) + self.stubs.Set(vmops.VMOps, 'reset_network', reset_network) stubs.stub_out_vm_methods(self.stubs) glance_stubs.stubout_glance_client(self.stubs) fake_utils.stub_out_utils_execute(self.stubs) @@ -211,7 +212,8 @@ class XenAPIVMTestCase(test.TestCase): 'ramdisk_id': 3, 'instance_type_id': '3', # m1.large 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': 'linux'} + 'os_type': 'linux', + 'architecture': 'x86-64'} instance = db.instance_create(self.context, values) self.conn.spawn(instance) @@ -228,6 +230,23 @@ class XenAPIVMTestCase(test.TestCase): instance = self._create_instance() self.conn.get_diagnostics(instance) + def test_instance_snapshot_fails_with_no_primary_vdi(self): + def create_bad_vbd(vm_ref, vdi_ref): + vbd_rec = {'VM': vm_ref, + 'VDI': vdi_ref, + 'userdevice': 'fake', + 'currently_attached': False} + vbd_ref = xenapi_fake._create_object('VBD', vbd_rec) + xenapi_fake.after_VBD_create(vbd_ref, vbd_rec) + return vbd_ref + + self.stubs.Set(xenapi_fake, 'create_vbd', create_bad_vbd) + stubs.stubout_instance_snapshot(self.stubs) + instance = self._create_instance() + + name = "MySnapshot" + self.assertRaises(exception.Error, self.conn.snapshot, instance, name) + def test_instance_snapshot(self): stubs.stubout_instance_snapshot(self.stubs) instance = self._create_instance() @@ -331,7 +350,7 @@ class XenAPIVMTestCase(test.TestCase): def check_vm_params_for_linux(self): self.assertEquals(self.vm['platform']['nx'], 'false') - self.assertEquals(self.vm['PV_args'], 'clocksource=jiffies') + self.assertEquals(self.vm['PV_args'], '') self.assertEquals(self.vm['PV_bootloader'], 'pygrub') # check that these are not set @@ -352,7 +371,8 @@ class XenAPIVMTestCase(test.TestCase): def _test_spawn(self, image_ref, kernel_id, ramdisk_id, instance_type_id="3", os_type="linux", - instance_id=1, check_injection=False): + architecture="x86-64", instance_id=1, + check_injection=False): stubs.stubout_loopingcall_start(self.stubs) values = {'id': instance_id, 'project_id': self.project.id, @@ -362,11 +382,14 @@ class XenAPIVMTestCase(test.TestCase): 'ramdisk_id': ramdisk_id, 'instance_type_id': instance_type_id, 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': os_type} + 'os_type': os_type, + 'architecture': architecture} instance = db.instance_create(self.context, values) self.conn.spawn(instance) self.create_vm_record(self.conn, os_type, instance_id) self.check_vm_record(self.conn, check_injection) + self.assertTrue(instance.os_type) + self.assertTrue(instance.architecture) def test_spawn_not_enough_memory(self): FLAGS.xenapi_image_service = 'glance' @@ -391,7 +414,7 @@ class XenAPIVMTestCase(test.TestCase): def test_spawn_vhd_glance_linux(self): FLAGS.xenapi_image_service = 'glance' self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, - os_type="linux") + os_type="linux", architecture="x86-64") self.check_vm_params_for_linux() def test_spawn_vhd_glance_swapdisk(self): @@ -420,7 +443,7 @@ class XenAPIVMTestCase(test.TestCase): def test_spawn_vhd_glance_windows(self): FLAGS.xenapi_image_service = 'glance' self._test_spawn(glance_stubs.FakeGlance.IMAGE_VHD, None, None, - os_type="windows") + os_type="windows", architecture="i386") self.check_vm_params_for_windows() def test_spawn_glance(self): @@ -571,7 +594,8 @@ class XenAPIVMTestCase(test.TestCase): 'ramdisk_id': 3, 'instance_type_id': '3', # m1.large 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': 'linux'} + 'os_type': 'linux', + 'architecture': 'x86-64'} instance = db.instance_create(self.context, values) self.conn.spawn(instance) return instance @@ -581,8 +605,8 @@ class XenAPIDiffieHellmanTestCase(test.TestCase): """Unit tests for Diffie-Hellman code.""" def setUp(self): super(XenAPIDiffieHellmanTestCase, self).setUp() - self.alice = SimpleDH() - self.bob = SimpleDH() + self.alice = vmops.SimpleDH() + self.bob = vmops.SimpleDH() def test_shared(self): alice_pub = self.alice.get_public() @@ -646,7 +670,8 @@ class XenAPIMigrateInstance(test.TestCase): 'local_gb': 5, 'instance_type_id': '3', # m1.large 'mac_address': 'aa:bb:cc:dd:ee:ff', - 'os_type': 'linux'} + 'os_type': 'linux', + 'architecture': 'x86-64'} fake_utils.stub_out_utils_execute(self.stubs) stubs.stub_out_migration_methods(self.stubs) @@ -685,6 +710,7 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): self.fake_instance = FakeInstance() self.fake_instance.id = 42 self.fake_instance.os_type = 'linux' + self.fake_instance.architecture = 'x86-64' def assert_disk_type(self, disk_type): dt = vm_utils.VMHelper.determine_disk_image_type( @@ -729,6 +755,28 @@ class XenAPIDetermineDiskImageTestCase(test.TestCase): self.assert_disk_type(vm_utils.ImageType.DISK_VHD) +class CompareVersionTestCase(test.TestCase): + def test_less_than(self): + """Test that cmp_version compares a as less than b""" + self.assertTrue(vmops.cmp_version('1.2.3.4', '1.2.3.5') < 0) + + def test_greater_than(self): + """Test that cmp_version compares a as greater than b""" + self.assertTrue(vmops.cmp_version('1.2.3.5', '1.2.3.4') > 0) + + def test_equal(self): + """Test that cmp_version compares a as equal to b""" + self.assertTrue(vmops.cmp_version('1.2.3.4', '1.2.3.4') == 0) + + def test_non_lexical(self): + """Test that cmp_version compares non-lexically""" + self.assertTrue(vmops.cmp_version('1.2.3.10', '1.2.3.4') > 0) + + def test_length(self): + """Test that cmp_version compares by length as last resort""" + self.assertTrue(vmops.cmp_version('1.2.3', '1.2.3.4') < 0) + + class FakeXenApi(object): """Fake XenApi for testing HostState.""" diff --git a/nova/utils.py b/nova/utils.py index 691134ada..e2ac16f31 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -35,6 +35,7 @@ import struct import sys import time import types +import uuid from xml.sax import saxutils from eventlet import event @@ -726,3 +727,17 @@ def parse_server_string(server_str): except: LOG.debug(_('Invalid server_string: %s' % server_str)) return ('', '') + + +def gen_uuid(): + return uuid.uuid4() + + +def is_uuid_like(val): + """For our purposes, a UUID is a string in canoical form: + + aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa + """ + if not isinstance(val, basestring): + return False + return (len(val) == 36) and (val.count('-') == 4) diff --git a/nova/virt/driver.py b/nova/virt/driver.py index eb9626d08..8dd7057a3 100644 --- a/nova/virt/driver.py +++ b/nova/virt/driver.py @@ -61,7 +61,7 @@ class ComputeDriver(object): """Return a list of InstanceInfo for all registered VMs""" raise NotImplementedError() - def spawn(self, instance, network_info=None): + def spawn(self, instance, network_info=None, block_device_mapping=None): """Launch a VM for the specified instance""" raise NotImplementedError() @@ -234,6 +234,10 @@ class ComputeDriver(object): """ raise NotImplementedError() + def agent_update(self, instance, url, md5hash): + """Update agent on the VM instance.""" + raise NotImplementedError() + def inject_network_info(self, instance): """inject network info for specified instance""" raise NotImplementedError() diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 0225797d7..94aecec24 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -129,7 +129,7 @@ class FakeConnection(driver.ComputeDriver): info_list.append(self._map_to_instance_info(instance)) return info_list - def spawn(self, instance): + def spawn(self, instance, network_info=None, block_device_mapping=None): """ Create a new instance/VM/domain on the virtualization platform. @@ -225,6 +225,21 @@ class FakeConnection(driver.ComputeDriver): """ pass + def agent_update(self, instance, url, md5hash): + """ + Update agent on the specified instance. + + The first parameter is an instance of nova.compute.service.Instance, + and so the instance is being specified as instance.name. The second + parameter is the URL of the agent to be fetched and updated on the + instance; the third is the md5 hash of the file for verification + purposes. + + The work will be done asynchronously. This function returns a + task that allows the caller to detect when it is complete. + """ + pass + def rescue(self, instance): """ Rescue the specified instance. @@ -237,6 +252,10 @@ class FakeConnection(driver.ComputeDriver): """ pass + def poll_rescued_instances(self, timeout): + """Poll for rescued instances""" + pass + def migrate_disk_and_power_off(self, instance, dest): """ Transfers the disk of a running instance in multiple phases, turning diff --git a/nova/virt/hyperv.py b/nova/virt/hyperv.py index 05b4775c1..772e7eb59 100644 --- a/nova/virt/hyperv.py +++ b/nova/virt/hyperv.py @@ -139,7 +139,7 @@ class HyperVConnection(driver.ComputeDriver): return instance_infos - def spawn(self, instance): + def spawn(self, instance, network_info=None, block_device_mapping=None): """ Create a new VM and start it.""" vm = self._lookup(instance.name) if vm is not None: diff --git a/nova/virt/libvirt.xml.template b/nova/virt/libvirt.xml.template index 20986d4d5..e1a683da8 100644 --- a/nova/virt/libvirt.xml.template +++ b/nova/virt/libvirt.xml.template @@ -67,11 +67,13 @@ <target dev='${disk_prefix}b' bus='${disk_bus}'/> </disk> #else + #if not ($getVar('ebs_root', False)) <disk type='file'> <driver type='${driver_type}'/> <source file='${basepath}/disk'/> <target dev='${disk_prefix}a' bus='${disk_bus}'/> </disk> + #end if #if $getVar('local', False) <disk type='file'> <driver type='${driver_type}'/> @@ -79,6 +81,13 @@ <target dev='${disk_prefix}b' bus='${disk_bus}'/> </disk> #end if + #for $vol in $volumes + <disk type='block'> + <driver type='raw'/> + <source dev='${vol.device_path}'/> + <target dev='${vol.mount_device}' bus='${disk_bus}'/> + </disk> + #end for #end if #end if diff --git a/nova/virt/libvirt/connection.py b/nova/virt/libvirt/connection.py index c68bdcbaf..2bb7415e8 100644 --- a/nova/virt/libvirt/connection.py +++ b/nova/virt/libvirt/connection.py @@ -41,6 +41,7 @@ import multiprocessing import netaddr import os import random +import re import shutil import subprocess import sys @@ -147,6 +148,10 @@ def _late_load_cheetah(): Template = t.Template +def _strip_dev(mount_path): + return re.sub(r'^/dev/', '', mount_path) + + class LibvirtConnection(driver.ComputeDriver): def __init__(self, read_only): @@ -574,11 +579,14 @@ class LibvirtConnection(driver.ComputeDriver): # NOTE(ilyaalekseyev): Implementation like in multinics # for xenapi(tr3buchet) @exception.wrap_exception - def spawn(self, instance, network_info=None): - xml = self.to_xml(instance, False, network_info) + def spawn(self, instance, network_info=None, block_device_mapping=None): + xml = self.to_xml(instance, False, network_info=network_info, + block_device_mapping=block_device_mapping) + block_device_mapping = block_device_mapping or [] self.firewall_driver.setup_basic_filtering(instance, network_info) self.firewall_driver.prepare_instance_filter(instance, network_info) - self._create_image(instance, xml, network_info=network_info) + self._create_image(instance, xml, network_info=network_info, + block_device_mapping=block_device_mapping) domain = self._create_new_domain(xml) LOG.debug(_("instance %s: is running"), instance['name']) self.firewall_driver.apply_instance_filter(instance) @@ -760,7 +768,8 @@ class LibvirtConnection(driver.ComputeDriver): # TODO(vish): should we format disk by default? def _create_image(self, inst, libvirt_xml, suffix='', disk_images=None, - network_info=None): + network_info=None, block_device_mapping=None): + block_device_mapping = block_device_mapping or [] if not network_info: network_info = netutils.get_network_info(inst) @@ -823,16 +832,19 @@ class LibvirtConnection(driver.ComputeDriver): size = None root_fname += "_sm" - self._cache_image(fn=self._fetch_image, - target=basepath('disk'), - fname=root_fname, - cow=FLAGS.use_cow_images, - image_id=disk_images['image_id'], - user=user, - project=project, - size=size) + if not self._volume_in_mapping(self.root_mount_device, + block_device_mapping): + self._cache_image(fn=self._fetch_image, + target=basepath('disk'), + fname=root_fname, + cow=FLAGS.use_cow_images, + image_id=disk_images['image_id'], + user=user, + project=project, + size=size) - if inst_type['local_gb']: + if inst_type['local_gb'] and not self._volume_in_mapping( + self.local_mount_device, block_device_mapping): self._cache_image(fn=self._create_local, target=basepath('disk.local'), fname="local_%s" % inst_type['local_gb'], @@ -947,7 +959,20 @@ class LibvirtConnection(driver.ComputeDriver): return result - def _prepare_xml_info(self, instance, rescue=False, network_info=None): + root_mount_device = 'vda' # FIXME for now. it's hard coded. + local_mount_device = 'vdb' # FIXME for now. it's hard coded. + + def _volume_in_mapping(self, mount_device, block_device_mapping): + mount_device_ = _strip_dev(mount_device) + for vol in block_device_mapping: + vol_mount_device = _strip_dev(vol['mount_device']) + if vol_mount_device == mount_device_: + return True + return False + + def _prepare_xml_info(self, instance, rescue=False, network_info=None, + block_device_mapping=None): + block_device_mapping = block_device_mapping or [] # TODO(adiantum) remove network_info creation code # when multinics will be completed if not network_info: @@ -965,6 +990,16 @@ class LibvirtConnection(driver.ComputeDriver): else: driver_type = 'raw' + for vol in block_device_mapping: + vol['mount_device'] = _strip_dev(vol['mount_device']) + ebs_root = self._volume_in_mapping(self.root_mount_device, + block_device_mapping) + if self._volume_in_mapping(self.local_mount_device, + block_device_mapping): + local_gb = False + else: + local_gb = inst_type['local_gb'] + xml_info = {'type': FLAGS.libvirt_type, 'name': instance['name'], 'basepath': os.path.join(FLAGS.instances_path, @@ -972,9 +1007,11 @@ class LibvirtConnection(driver.ComputeDriver): 'memory_kb': inst_type['memory_mb'] * 1024, 'vcpus': inst_type['vcpus'], 'rescue': rescue, - 'local': inst_type['local_gb'], + 'local': local_gb, 'driver_type': driver_type, - 'nics': nics} + 'nics': nics, + 'ebs_root': ebs_root, + 'volumes': block_device_mapping} if FLAGS.vnc_enabled: if FLAGS.libvirt_type != 'lxc': @@ -990,10 +1027,13 @@ class LibvirtConnection(driver.ComputeDriver): xml_info['disk'] = xml_info['basepath'] + "/disk" return xml_info - def to_xml(self, instance, rescue=False, network_info=None): + def to_xml(self, instance, rescue=False, network_info=None, + block_device_mapping=None): + block_device_mapping = block_device_mapping or [] # TODO(termie): cache? LOG.debug(_('instance %s: starting toXML method'), instance['name']) - xml_info = self._prepare_xml_info(instance, rescue, network_info) + xml_info = self._prepare_xml_info(instance, rescue, network_info, + block_device_mapping) xml = str(Template(self.libvirt_xml, searchList=[xml_info])) LOG.debug(_('instance %s: finished toXML method'), instance['name']) return xml diff --git a/nova/virt/vmwareapi/vmware_images.py b/nova/virt/vmwareapi/vmware_images.py index 48edc5384..70adba74f 100644 --- a/nova/virt/vmwareapi/vmware_images.py +++ b/nova/virt/vmwareapi/vmware_images.py @@ -90,8 +90,6 @@ def fetch_image(image, instance, **kwargs): func = _get_glance_image
elif FLAGS.image_service == "nova.image.s3.S3ImageService":
func = _get_s3_image
- elif FLAGS.image_service == "nova.image.local.LocalImageService":
- func = _get_local_image
else:
raise NotImplementedError(_("The Image Service %s is not implemented")
% FLAGS.image_service)
@@ -105,8 +103,6 @@ def upload_image(image, instance, **kwargs): func = _put_glance_image
elif FLAGS.image_service == "nova.image.s3.S3ImageService":
func = _put_s3_image
- elif FLAGS.image_service == "nova.image.local.LocalImageService":
- func = _put_local_image
else:
raise NotImplementedError(_("The Image Service %s is not implemented")
% FLAGS.image_service)
@@ -192,8 +188,6 @@ def get_vmdk_size_and_properties(image, instance): size, properties = meta_data["size"], meta_data["properties"]
elif FLAGS.image_service == "nova.image.s3.S3ImageService":
raise NotImplementedError
- elif FLAGS.image_service == "nova.image.local.LocalImageService":
- raise NotImplementedError
LOG.debug(_("Got image size of %(size)s for the image %(image)s") %
locals())
return size, properties
diff --git a/nova/virt/vmwareapi_conn.py b/nova/virt/vmwareapi_conn.py index 1c6d2572d..3c6345ec8 100644 --- a/nova/virt/vmwareapi_conn.py +++ b/nova/virt/vmwareapi_conn.py @@ -124,7 +124,7 @@ class VMWareESXConnection(driver.ComputeDriver): """List VM instances."""
return self._vmops.list_instances()
- def spawn(self, instance):
+ def spawn(self, instance, network_info=None, block_device_mapping=None):
"""Create VM instance."""
self._vmops.spawn(instance)
diff --git a/nova/virt/xenapi/fake.py b/nova/virt/xenapi/fake.py index 113198689..d5ac39473 100644 --- a/nova/virt/xenapi/fake.py +++ b/nova/virt/xenapi/fake.py @@ -146,6 +146,7 @@ def create_vdi(name_label, read_only, sr_ref, sharable): def create_vbd(vm_ref, vdi_ref): vbd_rec = {'VM': vm_ref, 'VDI': vdi_ref, + 'userdevice': '0', 'currently_attached': False} vbd_ref = _create_object('VBD', vbd_rec) after_VBD_create(vbd_ref, vbd_rec) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index b9d4346e4..f91958c57 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -157,7 +157,6 @@ class VMHelper(HelperBase): rec['PV_ramdisk'] = ramdisk else: # 2. Use kernel within the image - rec['PV_args'] = 'clocksource=jiffies' rec['PV_bootloader'] = 'pygrub' else: # 3. Using hardware virtualization @@ -284,19 +283,16 @@ class VMHelper(HelperBase): @classmethod def get_vdi_for_vm_safely(cls, session, vm_ref): - vdi_refs = VMHelper.lookup_vm_vdis(session, vm_ref) - if vdi_refs is None: - raise Exception(_("No VDIs found for VM %s") % vm_ref) - else: - num_vdis = len(vdi_refs) - if num_vdis != 1: - raise Exception( - _("Unexpected number of VDIs (%(num_vdis)s) found" - " for VM %(vm_ref)s") % locals()) - - vdi_ref = vdi_refs[0] - vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref) - return vdi_ref, vdi_rec + """Retrieves the primary VDI for a VM""" + vbd_refs = session.get_xenapi().VM.get_VBDs(vm_ref) + for vbd in vbd_refs: + vbd_rec = session.get_xenapi().VBD.get_record(vbd) + # Convention dictates the primary VDI will be userdevice 0 + if vbd_rec['userdevice'] == '0': + vdi_rec = session.get_xenapi().VDI.get_record(vbd_rec['VDI']) + return vbd_rec['VDI'], vdi_rec + raise exception.Error(_("No primary VDI found for" + "%(vm_ref)s") % locals()) @classmethod def create_snapshot(cls, session, instance_id, vm_ref, label): @@ -330,12 +326,6 @@ class VMHelper(HelperBase): return template_vm_ref, template_vdi_uuids @classmethod - def get_sr(cls, session, sr_label='slices'): - """Finds the SR named by the given name label and returns - the UUID""" - return session.call_xenapi('SR.get_by_name_label', sr_label)[0] - - @classmethod def get_sr_path(cls, session): """Return the path to our storage repository @@ -790,8 +780,7 @@ class VMHelper(HelperBase): @classmethod def scan_default_sr(cls, session): """Looks for the system default SR and triggers a re-scan""" - #FIXME(sirp/mdietz): refactor scan_default_sr in there - sr_ref = cls.get_sr(session) + sr_ref = find_sr(session) session.call_xenapi('SR.scan', sr_ref) @@ -883,7 +872,8 @@ def get_vdi_for_vm_safely(session, vm_ref): else: num_vdis = len(vdi_refs) if num_vdis != 1: - raise Exception(_("Unexpected number of VDIs (%(num_vdis)s) found" + raise exception.Exception(_("Unexpected number of VDIs" + "(%(num_vdis)s) found" " for VM %(vm_ref)s") % locals()) vdi_ref = vdi_refs[0] diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index c6d2b0936..2f4286184 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -47,6 +47,21 @@ LOG = logging.getLogger("nova.virt.xenapi.vmops") FLAGS = flags.FLAGS +def cmp_version(a, b): + """Compare two version strings (eg 0.0.1.10 > 0.0.1.9)""" + a = a.split('.') + b = b.split('.') + + # Compare each individual portion of both version strings + for va, vb in zip(a, b): + ret = int(va) - int(vb) + if ret: + return ret + + # Fallback to comparing length last + return len(a) - len(b) + + class VMOps(object): """ Management class for VM-related tasks @@ -160,9 +175,24 @@ class VMOps(object): # Create the VM ref and attach the first disk first_vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdis[0]['vdi_uuid']) - use_pv_kernel = VMHelper.determine_is_pv(self._session, - instance.id, first_vdi_ref, disk_image_type, - instance.os_type) + + vm_mode = instance.vm_mode and instance.vm_mode.lower() + if vm_mode == 'pv': + use_pv_kernel = True + elif vm_mode in ('hv', 'hvm'): + use_pv_kernel = False + vm_mode = 'hvm' # Normalize + else: + use_pv_kernel = VMHelper.determine_is_pv(self._session, + instance.id, first_vdi_ref, disk_image_type, + instance.os_type) + vm_mode = use_pv_kernel and 'pv' or 'hvm' + + if instance.vm_mode != vm_mode: + # Update database with normalized (or determined) value + db.instance_update(context.get_admin_context(), + instance['id'], {'vm_mode': vm_mode}) + vm_ref = VMHelper.create_vm(self._session, instance, kernel, ramdisk, use_pv_kernel) VMHelper.create_vbd(session=self._session, vm_ref=vm_ref, @@ -203,6 +233,34 @@ class VMOps(object): LOG.info(_('Spawning VM %(instance_name)s created %(vm_ref)s.') % locals()) + ctx = context.get_admin_context() + agent_build = db.agent_build_get_by_triple(ctx, 'xen', + instance.os_type, instance.architecture) + if agent_build: + LOG.info(_('Latest agent build for %(hypervisor)s/%(os)s' + \ + '/%(architecture)s is %(version)s') % agent_build) + else: + LOG.info(_('No agent build found for %(hypervisor)s/%(os)s' + \ + '/%(architecture)s') % { + 'hypervisor': 'xen', + 'os': instance.os_type, + 'architecture': instance.architecture}) + + def _check_agent_version(): + version = self.get_agent_version(instance) + if not version: + LOG.info(_('No agent version returned by instance')) + return + + LOG.info(_('Instance agent version: %s') % version) + if not agent_build: + return + + if cmp_version(version, agent_build['version']) < 0: + LOG.info(_('Updating Agent to %s') % agent_build['version']) + self.agent_update(instance, agent_build['url'], + agent_build['md5hash']) + def _inject_files(): injected_files = instance.injected_files if injected_files: @@ -237,6 +295,7 @@ class VMOps(object): if state == power_state.RUNNING: LOG.debug(_('Instance %s: booted'), instance_name) timer.stop() + _check_agent_version() _inject_files() _set_admin_password() return True @@ -443,6 +502,34 @@ class VMOps(object): task = self._session.call_xenapi('Async.VM.clean_reboot', vm_ref) self._session.wait_for_task(task, instance.id) + def get_agent_version(self, instance): + """Get the version of the agent running on the VM instance.""" + + # Send the encrypted password + transaction_id = str(uuid.uuid4()) + args = {'id': transaction_id} + resp = self._make_agent_call('version', instance, '', args) + if resp is None: + # No response from the agent + return + resp_dict = json.loads(resp) + return resp_dict['message'] + + def agent_update(self, instance, url, md5sum): + """Update agent on the VM instance.""" + + # Send the encrypted password + transaction_id = str(uuid.uuid4()) + args = {'id': transaction_id, 'url': url, 'md5sum': md5sum} + resp = self._make_agent_call('agentupdate', instance, '', args) + if resp is None: + # No response from the agent + return + resp_dict = json.loads(resp) + if resp_dict['returncode'] != '0': + raise RuntimeError(resp_dict['message']) + return resp_dict['message'] + def set_admin_password(self, instance, new_pass): """Set the root/admin password on the VM instance. diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index 6d828e109..5fcec1715 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -194,7 +194,7 @@ class XenAPIConnection(driver.ComputeDriver): def list_instances_detail(self): return self._vmops.list_instances_detail() - def spawn(self, instance): + def spawn(self, instance, network_info=None, block_device_mapping=None): """Create VM instance""" self._vmops.spawn(instance) diff --git a/nova/volume/api.py b/nova/volume/api.py index b07f2e94b..7d27abff9 100644 --- a/nova/volume/api.py +++ b/nova/volume/api.py @@ -21,6 +21,9 @@ Handles all requests relating to volumes. """ +from eventlet import greenthread + +from nova import db from nova import exception from nova import flags from nova import log as logging @@ -44,7 +47,8 @@ class API(base.Base): if snapshot['status'] != "available": raise exception.ApiError( _("Snapshot status must be available")) - size = snapshot['volume_size'] + if not size: + size = snapshot['volume_size'] if quota.allowed_volumes(context, 1, size) < 1: pid = context.project_id @@ -73,6 +77,14 @@ class API(base.Base): "snapshot_id": snapshot_id}}) return volume + # TODO(yamahata): eliminate dumb polling + def wait_creation(self, context, volume_id): + while True: + volume = self.get(context, volume_id) + if volume['status'] != 'creating': + return + greenthread.sleep(1) + def delete(self, context, volume_id): volume = self.get(context, volume_id) if volume['status'] != "available": diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 87e13277f..23e845deb 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -582,6 +582,14 @@ class FakeISCSIDriver(ISCSIDriver): """No setup necessary in fake mode.""" pass + def discover_volume(self, context, volume): + """Discover volume on a remote host.""" + return "/dev/disk/by-path/volume-id-%d" % volume['id'] + + def undiscover_volume(self, volume): + """Undiscover volume on a remote host.""" + pass + @staticmethod def fake_execute(cmd, *_args, **_kwargs): """Execute that simply logs the command.""" diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent index 9e761f264..b8a1b936a 100755 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/agent @@ -53,6 +53,19 @@ class TimeoutError(StandardError): pass +def version(self, arg_dict): + """Get version of agent.""" + arg_dict["value"] = json.dumps({"name": "version", "value": ""}) + request_id = arg_dict["id"] + arg_dict["path"] = "data/host/%s" % request_id + xenstore.write_record(self, arg_dict) + try: + resp = _wait_for_agent(self, request_id, arg_dict) + except TimeoutError, e: + raise PluginError(e) + return resp + + def key_init(self, arg_dict): """Handles the Diffie-Hellman key exchange with the agent to establish the shared secret key used to encrypt/decrypt sensitive @@ -144,6 +157,23 @@ def inject_file(self, arg_dict): return resp +def agent_update(self, arg_dict): + """Expects an URL and md5sum of the contents, then directs the agent to + update itself.""" + request_id = arg_dict["id"] + url = arg_dict["url"] + md5sum = arg_dict["md5sum"] + arg_dict["value"] = json.dumps({"name": "agentupdate", + "value": {"url": url, "md5sum": md5sum}}) + arg_dict["path"] = "data/host/%s" % request_id + xenstore.write_record(self, arg_dict) + try: + resp = _wait_for_agent(self, request_id, arg_dict) + except TimeoutError, e: + raise PluginError(e) + return resp + + def _agent_has_method(self, method): """Check that the agent has a particular method by checking its features. Cache the features so we don't have to query the agent @@ -201,7 +231,9 @@ def _wait_for_agent(self, request_id, arg_dict): if __name__ == "__main__": XenAPIPlugin.dispatch( - {"key_init": key_init, + {"version": version, + "key_init": key_init, "password": password, "resetnetwork": resetnetwork, - "inject_file": inject_file}) + "inject_file": inject_file, + "agentupdate": agent_update}) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration index 75c653408..ac1c50ad9 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/migration @@ -44,7 +44,7 @@ def move_vhds_into_sr(session, args): new_cow_uuid = params['new_cow_uuid'] sr_path = params['sr_path'] - sr_temp_path = "%s/images/" % sr_path + sr_temp_path = "%s/tmp/" % sr_path # Discover the copied VHDs locally, and then set up paths to copy # them to under the SR diff --git a/run_tests.py b/run_tests.py index d5d8acd16..bb33f9139 100644 --- a/run_tests.py +++ b/run_tests.py @@ -56,9 +56,11 @@ To run a single test module: """ import gettext +import heapq import os import unittest import sys +import time gettext.install('nova', unicode=1) @@ -183,9 +185,21 @@ class _NullColorizer(object): self.stream.write(text) +def get_elapsed_time_color(elapsed_time): + if elapsed_time > 1.0: + return 'red' + elif elapsed_time > 0.25: + return 'yellow' + else: + return 'green' + + class NovaTestResult(result.TextTestResult): def __init__(self, *args, **kw): + self.show_elapsed = kw.pop('show_elapsed') result.TextTestResult.__init__(self, *args, **kw) + self.num_slow_tests = 5 + self.slow_tests = [] # this is a fixed-sized heap self._last_case = None self.colorizer = None # NOTE(vish): reset stdout for the terminal check @@ -197,28 +211,49 @@ class NovaTestResult(result.TextTestResult): break sys.stdout = stdout + # NOTE(lorinh): Initialize start_time in case a sqlalchemy-migrate + # error results in it failing to be initialized later. Otherwise, + # _handleElapsedTime will fail, causing the wrong error message to + # be outputted. + self.start_time = time.time() + def getDescription(self, test): return str(test) - # NOTE(vish): copied from unittest with edit to add color - def addSuccess(self, test): - unittest.TestResult.addSuccess(self, test) + def _handleElapsedTime(self, test): + self.elapsed_time = time.time() - self.start_time + item = (self.elapsed_time, test) + # Record only the n-slowest tests using heap + if len(self.slow_tests) >= self.num_slow_tests: + heapq.heappushpop(self.slow_tests, item) + else: + heapq.heappush(self.slow_tests, item) + + def _writeElapsedTime(self, test): + color = get_elapsed_time_color(self.elapsed_time) + self.colorizer.write(" %.2f" % self.elapsed_time, color) + + def _writeResult(self, test, long_result, color, short_result, success): if self.showAll: - self.colorizer.write("OK", 'green') + self.colorizer.write(long_result, color) + if self.show_elapsed and success: + self._writeElapsedTime(test) self.stream.writeln() elif self.dots: - self.stream.write('.') + self.stream.write(short_result) self.stream.flush() # NOTE(vish): copied from unittest with edit to add color + def addSuccess(self, test): + unittest.TestResult.addSuccess(self, test) + self._handleElapsedTime(test) + self._writeResult(test, 'OK', 'green', '.', True) + + # NOTE(vish): copied from unittest with edit to add color def addFailure(self, test, err): unittest.TestResult.addFailure(self, test, err) - if self.showAll: - self.colorizer.write("FAIL", 'red') - self.stream.writeln() - elif self.dots: - self.stream.write('F') - self.stream.flush() + self._handleElapsedTime(test) + self._writeResult(test, 'FAIL', 'red', 'F', False) # NOTE(vish): copied from nose with edit to add color def addError(self, test, err): @@ -226,6 +261,7 @@ class NovaTestResult(result.TextTestResult): errorClasses. If the exception is a registered class, the error will be added to the list for that class, not errors. """ + self._handleElapsedTime(test) stream = getattr(self, 'stream', None) ec, ev, tb = err try: @@ -252,14 +288,11 @@ class NovaTestResult(result.TextTestResult): self.errors.append((test, exc_info)) test.passed = False if stream is not None: - if self.showAll: - self.colorizer.write("ERROR", 'red') - self.stream.writeln() - elif self.dots: - stream.write('E') + self._writeResult(test, 'ERROR', 'red', 'E', False) def startTest(self, test): unittest.TestResult.startTest(self, test) + self.start_time = time.time() current_case = test.test.__class__.__name__ if self.showAll: @@ -273,21 +306,47 @@ class NovaTestResult(result.TextTestResult): class NovaTestRunner(core.TextTestRunner): + def __init__(self, *args, **kwargs): + self.show_elapsed = kwargs.pop('show_elapsed') + core.TextTestRunner.__init__(self, *args, **kwargs) + def _makeResult(self): return NovaTestResult(self.stream, self.descriptions, self.verbosity, - self.config) + self.config, + show_elapsed=self.show_elapsed) + + def _writeSlowTests(self, result_): + # Pare out 'fast' tests + slow_tests = [item for item in result_.slow_tests + if get_elapsed_time_color(item[0]) != 'green'] + if slow_tests: + slow_total_time = sum(item[0] for item in slow_tests) + self.stream.writeln("Slowest %i tests took %.2f secs:" + % (len(slow_tests), slow_total_time)) + for elapsed_time, test in sorted(slow_tests, reverse=True): + time_str = "%.2f" % elapsed_time + self.stream.writeln(" %s %s" % (time_str.ljust(10), test)) + + def run(self, test): + result_ = core.TextTestRunner.run(self, test) + if self.show_elapsed: + self._writeSlowTests(result_) + return result_ if __name__ == '__main__': logging.setup() # If any argument looks like a test name but doesn't have "nova.tests" in # front of it, automatically add that so we don't have to type as much + show_elapsed = True argv = [] for x in sys.argv: if x.startswith('test_'): argv.append('nova.tests.%s' % x) + elif x.startswith('--hide-elapsed'): + show_elapsed = False else: argv.append(x) @@ -300,5 +359,6 @@ if __name__ == '__main__': runner = NovaTestRunner(stream=c.stream, verbosity=c.verbosity, - config=c) + config=c, + show_elapsed=show_elapsed) sys.exit(not core.run(config=c, testRunner=runner, argv=argv)) diff --git a/run_tests.sh b/run_tests.sh index c7bcd5d67..c3f06f837 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -10,6 +10,7 @@ function usage { echo " -f, --force Force a clean re-build of the virtual environment. Useful when dependencies have been added." echo " -p, --pep8 Just run pep8" echo " -h, --help Print this usage message" + echo " --hide-elapsed Don't print the elapsed time for each test along with slow test list" echo "" echo "Note: with no options specified, the script will try to run the tests in a virtual environment," echo " If no virtualenv is found, the script will ask if you would like to create one. If you " @@ -24,6 +25,7 @@ function process_option { -N|--no-virtual-env) let always_venv=0; let never_venv=1;; -f|--force) let force=1;; -p|--pep8) let just_pep8=1;; + -*) noseopts="$noseopts $1";; *) noseargs="$noseargs $1" esac } @@ -34,6 +36,7 @@ always_venv=0 never_venv=0 force=0 noseargs= +noseopts= wrapper="" just_pep8=0 @@ -72,7 +75,7 @@ function run_pep8 { --exclude=vcsversion.py ${srcfiles} } -NOSETESTS="python run_tests.py $noseargs" +NOSETESTS="python run_tests.py $noseopts $noseargs" if [ $never_venv -eq 0 ] then @@ -107,7 +110,10 @@ fi run_tests || exit -# Also run pep8 if no options were provided. +# NOTE(sirp): we only want to run pep8 when we're running the full-test suite, +# not when we're running tests individually. To handle this, we need to +# distinguish between options (noseopts), which begin with a '-', and +# arguments (noseargs). if [ -z "$noseargs" ]; then run_pep8 fi diff --git a/tools/pip-requires b/tools/pip-requires index af8ce80cd..5d31a814d 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -9,12 +9,12 @@ boto==1.9b carrot==0.10.5 eventlet==0.9.12 lockfile==0.8 -python-novaclient==2.3 +python-novaclient==2.5.3 python-daemon==1.5.5 python-gflags==1.3 redis==2.0.0 routes==1.12.3 -WebOb==0.9.8 +WebOb==1.0.8 wsgiref==0.1.2 mox==0.5.3 greenlet==0.3.1 |
