Pulled trunk, merged boot from ISO changes.

17 files changed, 1362 insertions, 839 deletions

diff --git a/nova/virt/connection.py b/nova/virt/connection.py
index 99a8849f1..aeec17c98 100644
--- a/nova/virt/connection.py
+++ b/nova/virt/connection.py
@@ -27,9 +27,9 @@ from nova import utils
 from nova.virt import driver
 from nova.virt import fake
 from nova.virt import hyperv
-from nova.virt import libvirt_conn
 from nova.virt import vmwareapi_conn
 from nova.virt import xenapi_conn
+from nova.virt.libvirt import connection as libvirt_conn
 
 
 LOG = logging.getLogger("nova.virt.connection")
diff --git a/nova/virt/driver.py b/nova/virt/driver.py
index eb9626d08..2c7c0cfcc 100644
--- a/nova/virt/driver.py
+++ b/nova/virt/driver.py
@@ -61,7 +61,7 @@ class ComputeDriver(object):
         """Return a list of InstanceInfo for all registered VMs"""
         raise NotImplementedError()
 
-    def spawn(self, instance, network_info=None):
+    def spawn(self, instance, network_info=None, block_device_mapping=None):
         """Launch a VM for the specified instance"""
         raise NotImplementedError()
 
@@ -191,6 +191,10 @@ class ComputeDriver(object):
     def refresh_security_group_members(self, security_group_id):
         raise NotImplementedError()
 
+    def refresh_provider_fw_rules(self, security_group_id):
+        """See: nova/virt/fake.py for docs."""
+        raise NotImplementedError()
+
     def reset_network(self, instance):
         """reset networking for specified instance"""
         raise NotImplementedError()
@@ -234,6 +238,10 @@ class ComputeDriver(object):
         """
         raise NotImplementedError()
 
+    def agent_update(self, instance, url, md5hash):
+        """Update agent on the VM instance."""
+        raise NotImplementedError()
+
     def inject_network_info(self, instance):
         """inject network info for specified instance"""
         raise NotImplementedError()
diff --git a/nova/virt/fake.py b/nova/virt/fake.py
index 5ac376e46..f78c29bd0 100644
--- a/nova/virt/fake.py
+++ b/nova/virt/fake.py
@@ -82,6 +82,21 @@ class FakeConnection(driver.ComputeDriver):
 
     def __init__(self):
         self.instances = {}
+        self.host_status = {
+          'host_name-description': 'Fake Host',
+          'host_hostname': 'fake-mini',
+          'host_memory_total': 8000000000,
+          'host_memory_overhead': 10000000,
+          'host_memory_free': 7900000000,
+          'host_memory_free_computed': 7900000000,
+          'host_other_config': {},
+          'host_ip_address': '192.168.1.109',
+          'host_cpu_info': {},
+          'disk_available': 500000000000,
+          'disk_total': 600000000000,
+          'disk_used': 100000000000,
+          'host_uuid': 'cedb9b39-9388-41df-8891-c5c9a0c0fe5f',
+          'host_name_label': 'fake-mini'}
 
     @classmethod
     def instance(cls):
@@ -114,7 +129,7 @@ class FakeConnection(driver.ComputeDriver):
             info_list.append(self._map_to_instance_info(instance))
         return info_list
 
-    def spawn(self, instance):
+    def spawn(self, instance, network_info=None, block_device_mapping=None):
         """
         Create a new instance/VM/domain on the virtualization platform.
 
@@ -210,6 +225,21 @@ class FakeConnection(driver.ComputeDriver):
         """
         pass
 
+    def agent_update(self, instance, url, md5hash):
+        """
+        Update agent on the specified instance.
+
+        The first parameter is an instance of nova.compute.service.Instance,
+        and so the instance is being specified as instance.name. The second
+        parameter is the URL of the agent to be fetched and updated on the
+        instance; the third is the md5 hash of the file for verification
+        purposes.
+
+        The work will be done asynchronously.  This function returns a
+        task that allows the caller to detect when it is complete.
+        """
+        pass
+
     def rescue(self, instance):
         """
         Rescue the specified instance.
@@ -222,6 +252,10 @@ class FakeConnection(driver.ComputeDriver):
         """
         pass
 
+    def poll_rescued_instances(self, timeout):
+        """Poll for rescued instances"""
+        pass
+
     def migrate_disk_and_power_off(self, instance, dest):
         """
         Transfers the disk of a running instance in multiple phases, turning
@@ -432,6 +466,22 @@ class FakeConnection(driver.ComputeDriver):
         """
         return True
 
+    def refresh_provider_fw_rules(self):
+        """This triggers a firewall update based on database changes.
+
+        When this is called, rules have either been added or removed from the
+        datastore.  You can retrieve rules with
+        :method:`nova.db.api.provider_fw_rule_get_all`.
+
+        Provider rules take precedence over security group rules.  If an IP
+        would be allowed by a security group ingress rule, but blocked by
+        a provider rule, then packets from the IP are dropped.  This includes
+        intra-project traffic in the case of the allow_project_net_traffic
+        flag for the libvirt-derived classes.
+
+        """
+        pass
+
     def update_available_resource(self, ctxt, host):
         """This method is supported only by libvirt."""
         return
@@ -456,3 +506,11 @@ class FakeConnection(driver.ComputeDriver):
     def test_remove_vm(self, instance_name):
         """ Removes the named VM, as if it crashed. For testing"""
         self.instances.pop(instance_name)
+
+    def update_host_status(self):
+        """Return fake Host Status of ram, disk, network."""
+        return self.host_status
+
+    def get_host_stats(self, refresh=False):
+        """Return fake Host Status of ram, disk, network."""
+        return self.host_status
diff --git a/nova/virt/hyperv.py b/nova/virt/hyperv.py
index 1142e97a4..772e7eb59 100644
--- a/nova/virt/hyperv.py
+++ b/nova/virt/hyperv.py
@@ -139,7 +139,7 @@ class HyperVConnection(driver.ComputeDriver):
 
         return instance_infos
 
-    def spawn(self, instance):
+    def spawn(self, instance, network_info=None, block_device_mapping=None):
         """ Create a new VM and start it."""
         vm = self._lookup(instance.name)
         if vm is not None:
@@ -151,7 +151,7 @@ class HyperVConnection(driver.ComputeDriver):
         base_vhd_filename = os.path.join(FLAGS.instances_path,
                                          instance.name)
         vhdfile = "%s.vhd" % (base_vhd_filename)
-        images.fetch(instance['image_id'], vhdfile, user, project)
+        images.fetch(instance['image_ref'], vhdfile, user, project)
 
         try:
             self._create_vm(instance)
diff --git a/nova/virt/images.py b/nova/virt/images.py
index 02c898fda..40bf6107c 100644
--- a/nova/virt/images.py
+++ b/nova/virt/images.py
@@ -23,6 +23,8 @@ Handling of VM disk images.
 
 from nova import context
 from nova import flags
+from nova.image import glance as glance_image_service
+import nova.image
 from nova import log as logging
 from nova import utils
 
@@ -31,23 +33,13 @@ FLAGS = flags.FLAGS
 LOG = logging.getLogger('nova.virt.images')
 
 
-def fetch(image_id, path, _user, _project):
+def fetch(image_href, path, _user, _project):
     # TODO(vish): Improve context handling and add owner and auth data
     #             when it is added to glance.  Right now there is no
     #             auth checking in glance, so we assume that access was
     #             checked before we got here.
-    image_service = utils.import_object(FLAGS.image_service)
+    (image_service, image_id) = nova.image.get_image_service(image_href)
     with open(path, "wb") as image_file:
         elevated = context.get_admin_context()
         metadata = image_service.get(elevated, image_id, image_file)
     return metadata
-
-
-# TODO(vish): xenapi should use the glance client code directly instead
-#             of retrieving the image using this method.
-def image_url(image):
-    if FLAGS.image_service == "nova.image.glance.GlanceImageService":
-        return "http://%s:%s/images/%s" % (FLAGS.glance_host,
-            FLAGS.glance_port, image)
-    return "http://%s:%s/_images/%s/image" % (FLAGS.s3_host, FLAGS.s3_port,
-                                              image)
diff --git a/nova/virt/libvirt.xml.template b/nova/virt/libvirt.xml.template
index de2497a76..e1a683da8 100644
--- a/nova/virt/libvirt.xml.template
+++ b/nova/virt/libvirt.xml.template
@@ -67,11 +67,13 @@
             <target dev='${disk_prefix}b' bus='${disk_bus}'/>
         </disk>
     #else
+        #if not ($getVar('ebs_root', False))
         <disk type='file'>
             <driver type='${driver_type}'/>
             <source file='${basepath}/disk'/>
             <target dev='${disk_prefix}a' bus='${disk_bus}'/>
         </disk>
+        #end if
         #if $getVar('local', False)
             <disk type='file'>
                 <driver type='${driver_type}'/>
@@ -79,6 +81,13 @@
                 <target dev='${disk_prefix}b' bus='${disk_bus}'/>
             </disk>
         #end if
+        #for $vol in $volumes
+            <disk type='block'>
+                <driver type='raw'/>
+                <source dev='${vol.device_path}'/>
+                <target dev='${vol.mount_device}' bus='${disk_bus}'/>
+            </disk>
+        #end for
     #end if
 #end if
 
@@ -116,7 +125,7 @@
         </serial>
 
 #if $getVar('vncserver_host', False)
-        <graphics type='vnc' port='-1' autoport='yes' keymap='en-us' listen='${vncserver_host}'/>
+        <graphics type='vnc' port='-1' autoport='yes' keymap='${vnc_keymap}' listen='${vncserver_host}'/>
 #end if
     </devices>
 </domain>
diff --git a/nova/virt/libvirt/__init__.py b/nova/virt/libvirt/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/nova/virt/libvirt/__init__.py
diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt/connection.py
index fa918b0a3..b06bfb714 100644
--- a/nova/virt/libvirt_conn.py
+++ b/nova/virt/libvirt/connection.py
@@ -36,9 +36,12 @@ Supports KVM, LXC, QEMU, UML, and XEN.
 
 """
 
+import hashlib
 import multiprocessing
+import netaddr
 import os
 import random
+import re
 import shutil
 import subprocess
 import sys
@@ -51,13 +54,11 @@ from xml.etree import ElementTree
 from eventlet import greenthread
 from eventlet import tpool
 
-import IPy
-
 from nova import context
 from nova import db
 from nova import exception
 from nova import flags
-from nova import ipv6
+import nova.image
 from nova import log as logging
 from nova import utils
 from nova import vnc
@@ -67,20 +68,23 @@ from nova.compute import power_state
 from nova.virt import disk
 from nova.virt import driver
 from nova.virt import images
+from nova.virt.libvirt import netutils
+
 
 libvirt = None
 libxml2 = None
 Template = None
 
+
 LOG = logging.getLogger('nova.virt.libvirt_conn')
 
+
 FLAGS = flags.FLAGS
 flags.DECLARE('live_migration_retry_count', 'nova.compute.manager')
 # TODO(vish): These flags should probably go into a shared location
 flags.DEFINE_string('rescue_image_id', 'ami-rescue', 'Rescue ami image')
 flags.DEFINE_string('rescue_kernel_id', 'aki-rescue', 'Rescue aki image')
 flags.DEFINE_string('rescue_ramdisk_id', 'ari-rescue', 'Rescue ari image')
-
 flags.DEFINE_string('libvirt_xml_template',
                     utils.abspath('virt/libvirt.xml.template'),
                     'Libvirt XML Template')
@@ -102,7 +106,7 @@ flags.DEFINE_string('ajaxterm_portrange',
                     '10000-12000',
                     'Range of ports that ajaxterm should randomly try to bind')
 flags.DEFINE_string('firewall_driver',
-                    'nova.virt.libvirt_conn.IptablesFirewallDriver',
+                    'nova.virt.libvirt.firewall.IptablesFirewallDriver',
                     'Firewall driver (defaults to iptables)')
 flags.DEFINE_string('cpuinfo_xml_template',
                     utils.abspath('virt/cpuinfo.xml.template'),
@@ -144,68 +148,8 @@ def _late_load_cheetah():
         Template = t.Template
 
 
-def _get_net_and_mask(cidr):
-    net = IPy.IP(cidr)
-    return str(net.net()), str(net.netmask())
-
-
-def _get_net_and_prefixlen(cidr):
-    net = IPy.IP(cidr)
-    return str(net.net()), str(net.prefixlen())
-
-
-def _get_ip_version(cidr):
-    net = IPy.IP(cidr)
-    return int(net.version())
-
-
-def _get_network_info(instance):
-    # TODO(adiantum) If we will keep this function
-    # we should cache network_info
-    admin_context = context.get_admin_context()
-
-    ip_addresses = db.fixed_ip_get_all_by_instance(admin_context,
-                                                   instance['id'])
-    networks = db.network_get_all_by_instance(admin_context,
-                                              instance['id'])
-    flavor = db.instance_type_get_by_id(admin_context,
-                                        instance['instance_type_id'])
-    network_info = []
-
-    for network in networks:
-        network_ips = [ip for ip in ip_addresses
-                       if ip['network_id'] == network['id']]
-
-        def ip_dict(ip):
-            return {
-                'ip': ip['address'],
-                'netmask': network['netmask'],
-                'enabled': '1'}
-
-        def ip6_dict():
-            prefix = network['cidr_v6']
-            mac = instance['mac_address']
-            project_id = instance['project_id']
-            return  {
-                'ip': ipv6.to_global(prefix, mac, project_id),
-                'netmask': network['netmask_v6'],
-                'enabled': '1'}
-
-        mapping = {
-            'label': network['label'],
-            'gateway': network['gateway'],
-            'broadcast': network['broadcast'],
-            'mac': instance['mac_address'],
-            'rxtx_cap': flavor['rxtx_cap'],
-            'dns': [network['dns']],
-            'ips': [ip_dict(ip) for ip in network_ips]}
-
-        if FLAGS.use_ipv6:
-            mapping['ip6s'] = [ip6_dict()]
-            mapping['gateway6'] = network['gateway_v6']
-
-        network_info.append((network, mapping))
-    return network_info
+def _strip_dev(mount_path):
+        return re.sub(r'^/dev/', '', mount_path)
 
 
 class LibvirtConnection(driver.ComputeDriver):
@@ -440,7 +384,7 @@ class LibvirtConnection(driver.ComputeDriver):
         virt_dom.detachDevice(xml)
 
     @exception.wrap_exception
-    def snapshot(self, instance, image_id):
+    def snapshot(self, instance, image_href):
         """Create snapshot from a running VM instance.
 
         This command only works with qemu 0.14+, the qemu_img flag is
@@ -448,17 +392,22 @@ class LibvirtConnection(driver.ComputeDriver):
         to support this command.
 
         """
-        image_service = utils.import_object(FLAGS.image_service)
         virt_dom = self._lookup_by_name(instance['name'])
         elevated = context.get_admin_context()
 
-        base = image_service.show(elevated, instance['image_id'])
+        (image_service, image_id) = nova.image.get_image_service(
+            instance['image_ref'])
+        base = image_service.show(elevated, image_id)
+        (snapshot_image_service, snapshot_image_id) = \
+            nova.image.get_image_service(image_href)
+        snapshot = snapshot_image_service.show(elevated, snapshot_image_id)
 
         metadata = {'disk_format': base['disk_format'],
                     'container_format': base['container_format'],
                     'is_public': False,
-                    'name': '%s.%s' % (base['name'], image_id),
-                    'properties': {'architecture': base['architecture'],
+                    'status': 'active',
+                    'name': snapshot['name'],
+                    'properties': {
                                    'kernel_id': instance['kernel_id'],
                                    'image_location': 'snapshot',
                                    'image_state': 'available',
@@ -466,6 +415,9 @@ class LibvirtConnection(driver.ComputeDriver):
                                    'ramdisk_id': instance['ramdisk_id'],
                                    }
                     }
+        if 'architecture' in base['properties']:
+            arch = base['properties']['architecture']
+            metadata['properties']['architecture'] = arch
 
         # Make the snapshot
         snapshot_name = uuid.uuid4().hex
@@ -500,7 +452,7 @@ class LibvirtConnection(driver.ComputeDriver):
         # Upload that image to the image service
         with open(out_path) as image_file:
             image_service.update(elevated,
-                                 image_id,
+                                 image_href,
                                  metadata,
                                  image_file)
 
@@ -550,19 +502,27 @@ class LibvirtConnection(driver.ComputeDriver):
 
     @exception.wrap_exception
     def pause(self, instance, callback):
-        raise exception.ApiError("pause not supported for libvirt.")
+        """Pause VM instance"""
+        dom = self._lookup_by_name(instance.name)
+        dom.suspend()
 
     @exception.wrap_exception
     def unpause(self, instance, callback):
-        raise exception.ApiError("unpause not supported for libvirt.")
+        """Unpause paused VM instance"""
+        dom = self._lookup_by_name(instance.name)
+        dom.resume()
 
     @exception.wrap_exception
     def suspend(self, instance, callback):
-        raise exception.ApiError("suspend not supported for libvirt")
+        """Suspend the specified instance"""
+        dom = self._lookup_by_name(instance.name)
+        dom.managedSave(0)
 
     @exception.wrap_exception
     def resume(self, instance, callback):
-        raise exception.ApiError("resume not supported for libvirt")
+        """resume the specified instance"""
+        dom = self._lookup_by_name(instance.name)
+        dom.create()
 
     @exception.wrap_exception
     def rescue(self, instance):
@@ -619,11 +579,14 @@ class LibvirtConnection(driver.ComputeDriver):
     # NOTE(ilyaalekseyev): Implementation like in multinics
     # for xenapi(tr3buchet)
     @exception.wrap_exception
-    def spawn(self, instance, network_info=None):
-        xml = self.to_xml(instance, False, network_info)
+    def spawn(self, instance, network_info=None, block_device_mapping=None):
+        xml = self.to_xml(instance, False, network_info=network_info,
+                          block_device_mapping=block_device_mapping)
+        block_device_mapping = block_device_mapping or []
         self.firewall_driver.setup_basic_filtering(instance, network_info)
         self.firewall_driver.prepare_instance_filter(instance, network_info)
-        self._create_image(instance, xml, network_info=network_info)
+        self._create_image(instance, xml, network_info=network_info,
+                           block_device_mapping=block_device_mapping)
         domain = self._create_new_domain(xml)
         LOG.debug(_("instance %s: is running"), instance['name'])
         self.firewall_driver.apply_instance_filter(instance)
@@ -805,9 +768,10 @@ class LibvirtConnection(driver.ComputeDriver):
         # TODO(vish): should we format disk by default?
 
     def _create_image(self, inst, libvirt_xml, suffix='', disk_images=None,
-                        network_info=None):
+                        network_info=None, block_device_mapping=None):
+        block_device_mapping = block_device_mapping or []
         if not network_info:
-            network_info = _get_network_info(inst)
+            network_info = netutils.get_network_info(inst)
 
         if not suffix:
             suffix = ''
@@ -838,7 +802,7 @@ class LibvirtConnection(driver.ComputeDriver):
         project = manager.AuthManager().get_project(inst['project_id'])
 
         if not disk_images:
-            disk_images = {'image_id': inst['image_id'],
+            disk_images = {'image_id': inst['image_ref'],
                            'kernel_id': inst['kernel_id'],
                            'ramdisk_id': inst['ramdisk_id']}
 
@@ -859,7 +823,7 @@ class LibvirtConnection(driver.ComputeDriver):
                                   user=user,
                                   project=project)
 
-        root_fname = '%08x' % int(disk_images['image_id'])
+        root_fname = hashlib.sha1(disk_images['image_id']).hexdigest()
         size = FLAGS.minimum_root_size
 
         inst_type_id = inst['instance_type_id']
@@ -868,16 +832,19 @@ class LibvirtConnection(driver.ComputeDriver):
             size = None
             root_fname += "_sm"
 
-        self._cache_image(fn=self._fetch_image,
-                          target=basepath('disk'),
-                          fname=root_fname,
-                          cow=FLAGS.use_cow_images,
-                          image_id=disk_images['image_id'],
-                          user=user,
-                          project=project,
-                          size=size)
+        if not self._volume_in_mapping(self.root_mount_device,
+                                       block_device_mapping):
+            self._cache_image(fn=self._fetch_image,
+                              target=basepath('disk'),
+                              fname=root_fname,
+                              cow=FLAGS.use_cow_images,
+                              image_id=disk_images['image_id'],
+                              user=user,
+                              project=project,
+                              size=size)
 
-        if inst_type['local_gb']:
+        if inst_type['local_gb'] and not self._volume_in_mapping(
+            self.local_mount_device, block_device_mapping):
             self._cache_image(fn=self._create_local,
                               target=basepath('disk.local'),
                               fname="local_%s" % inst_type['local_gb'],
@@ -934,7 +901,7 @@ class LibvirtConnection(driver.ComputeDriver):
 
         if key or net:
             inst_name = inst['name']
-            img_id = inst.image_id
+            img_id = inst.image_ref
             if key:
                 LOG.info(_('instance %(inst_name)s: injecting key into'
                         ' image %(img_id)s') % locals())
@@ -966,10 +933,10 @@ class LibvirtConnection(driver.ComputeDriver):
 
         if FLAGS.allow_project_net_traffic:
             template = "<parameter name=\"%s\"value=\"%s\" />\n"
-            net, mask = _get_net_and_mask(network['cidr'])
+            net, mask = netutils.get_net_and_mask(network['cidr'])
             values = [("PROJNET", net), ("PROJMASK", mask)]
             if FLAGS.use_ipv6:
-                net_v6, prefixlen_v6 = _get_net_and_prefixlen(
+                net_v6, prefixlen_v6 = netutils.get_net_and_prefixlen(
                                            network['cidr_v6'])
                 values.extend([("PROJNETV6", net_v6),
                                ("PROJMASKV6", prefixlen_v6)])
@@ -992,11 +959,24 @@ class LibvirtConnection(driver.ComputeDriver):
 
         return result
 
-    def _prepare_xml_info(self, instance, rescue=False, network_info=None):
+    root_mount_device = 'vda'  # FIXME for now. it's hard coded.
+    local_mount_device = 'vdb'  # FIXME for now. it's hard coded.
+
+    def _volume_in_mapping(self, mount_device, block_device_mapping):
+        mount_device_ = _strip_dev(mount_device)
+        for vol in block_device_mapping:
+            vol_mount_device = _strip_dev(vol['mount_device'])
+            if vol_mount_device == mount_device_:
+                return True
+        return False
+
+    def _prepare_xml_info(self, instance, rescue=False, network_info=None,
+                          block_device_mapping=None):
+        block_device_mapping = block_device_mapping or []
         # TODO(adiantum) remove network_info creation code
         # when multinics will be completed
         if not network_info:
-            network_info = _get_network_info(instance)
+            network_info = netutils.get_network_info(instance)
 
         nics = []
         for (network, mapping) in network_info:
@@ -1010,6 +990,16 @@ class LibvirtConnection(driver.ComputeDriver):
         else:
             driver_type = 'raw'
 
+        for vol in block_device_mapping:
+            vol['mount_device'] = _strip_dev(vol['mount_device'])
+        ebs_root = self._volume_in_mapping(self.root_mount_device,
+                                           block_device_mapping)
+        if self._volume_in_mapping(self.local_mount_device,
+                                   block_device_mapping):
+            local_gb = False
+        else:
+            local_gb = inst_type['local_gb']
+
         xml_info = {'type': FLAGS.libvirt_type,
                     'name': instance['name'],
                     'basepath': os.path.join(FLAGS.instances_path,
@@ -1017,13 +1007,16 @@ class LibvirtConnection(driver.ComputeDriver):
                     'memory_kb': inst_type['memory_mb'] * 1024,
                     'vcpus': inst_type['vcpus'],
                     'rescue': rescue,
-                    'local': inst_type['local_gb'],
+                    'local': local_gb,
                     'driver_type': driver_type,
-                    'nics': nics}
+                    'nics': nics,
+                    'ebs_root': ebs_root,
+                    'volumes': block_device_mapping}
 
         if FLAGS.vnc_enabled:
             if FLAGS.libvirt_type != 'lxc':
                 xml_info['vncserver_host'] = FLAGS.vncserver_host
+                xml_info['vnc_keymap'] = FLAGS.vnc_keymap
         if not rescue:
             if instance['kernel_id']:
                 xml_info['kernel'] = xml_info['basepath'] + "/kernel"
@@ -1034,10 +1027,13 @@ class LibvirtConnection(driver.ComputeDriver):
             xml_info['disk'] = xml_info['basepath'] + "/disk"
         return xml_info
 
-    def to_xml(self, instance, rescue=False, network_info=None):
+    def to_xml(self, instance, rescue=False, network_info=None,
+               block_device_mapping=None):
+        block_device_mapping = block_device_mapping or []
         # TODO(termie): cache?
         LOG.debug(_('instance %s: starting toXML method'), instance['name'])
-        xml_info = self._prepare_xml_info(instance, rescue, network_info)
+        xml_info = self._prepare_xml_info(instance, rescue, network_info,
+                                          block_device_mapping)
         xml = str(Template(self.libvirt_xml, searchList=[xml_info]))
         LOG.debug(_('instance %s: finished toXML method'), instance['name'])
         return xml
@@ -1386,6 +1382,9 @@ class LibvirtConnection(driver.ComputeDriver):
     def refresh_security_group_members(self, security_group_id):
         self.firewall_driver.refresh_security_group_members(security_group_id)
 
+    def refresh_provider_fw_rules(self):
+        self.firewall_driver.refresh_provider_fw_rules()
+
     def update_available_resource(self, ctxt, host):
         """Updates compute manager resource info on ComputeNode table.
 
@@ -1591,606 +1590,3 @@ class LibvirtConnection(driver.ComputeDriver):
     def get_host_stats(self, refresh=False):
         """See xenapi_conn.py implementation."""
         pass
-
-
-class FirewallDriver(object):
-    def prepare_instance_filter(self, instance, network_info=None):
-        """Prepare filters for the instance.
-
-        At this point, the instance isn't running yet."""
-        raise NotImplementedError()
-
-    def unfilter_instance(self, instance):
-        """Stop filtering instance"""
-        raise NotImplementedError()
-
-    def apply_instance_filter(self, instance):
-        """Apply instance filter.
-
-        Once this method returns, the instance should be firewalled
-        appropriately. This method should as far as possible be a
-        no-op. It's vastly preferred to get everything set up in
-        prepare_instance_filter.
-        """
-        raise NotImplementedError()
-
-    def refresh_security_group_rules(self,
-                                     security_group_id,
-                                     network_info=None):
-        """Refresh security group rules from data store
-
-        Gets called when a rule has been added to or removed from
-        the security group."""
-        raise NotImplementedError()
-
-    def refresh_security_group_members(self, security_group_id):
-        """Refresh security group members from data store
-
-        Gets called when an instance gets added to or removed from
-        the security group."""
-        raise NotImplementedError()
-
-    def setup_basic_filtering(self, instance, network_info=None):
-        """Create rules to block spoofing and allow dhcp.
-
-        This gets called when spawning an instance, before
-        :method:`prepare_instance_filter`.
-
-        """
-        raise NotImplementedError()
-
-    def instance_filter_exists(self, instance):
-        """Check nova-instance-instance-xxx exists"""
-        raise NotImplementedError()
-
-
-class NWFilterFirewall(FirewallDriver):
-    """
-    This class implements a network filtering mechanism versatile
-    enough for EC2 style Security Group filtering by leveraging
-    libvirt's nwfilter.
-
-    First, all instances get a filter ("nova-base-filter") applied.
-    This filter provides some basic security such as protection against
-    MAC spoofing, IP spoofing, and ARP spoofing.
-
-    This filter drops all incoming ipv4 and ipv6 connections.
-    Outgoing connections are never blocked.
-
-    Second, every security group maps to a nwfilter filter(*).
-    NWFilters can be updated at runtime and changes are applied
-    immediately, so changes to security groups can be applied at
-    runtime (as mandated by the spec).
-
-    Security group rules are named "nova-secgroup-<id>" where <id>
-    is the internal id of the security group. They're applied only on
-    hosts that have instances in the security group in question.
-
-    Updates to security groups are done by updating the data model
-    (in response to API calls) followed by a request sent to all
-    the nodes with instances in the security group to refresh the
-    security group.
-
-    Each instance has its own NWFilter, which references the above
-    mentioned security group NWFilters. This was done because
-    interfaces can only reference one filter while filters can
-    reference multiple other filters. This has the added benefit of
-    actually being able to add and remove security groups from an
-    instance at run time. This functionality is not exposed anywhere,
-    though.
-
-    Outstanding questions:
-
-    The name is unique, so would there be any good reason to sync
-    the uuid across the nodes (by assigning it from the datamodel)?
-
-
-    (*) This sentence brought to you by the redundancy department of
-        redundancy.
-
-    """
-
-    def __init__(self, get_connection, **kwargs):
-        self._libvirt_get_connection = get_connection
-        self.static_filters_configured = False
-        self.handle_security_groups = False
-
-    def apply_instance_filter(self, instance):
-        """No-op. Everything is done in prepare_instance_filter"""
-        pass
-
-    def _get_connection(self):
-        return self._libvirt_get_connection()
-    _conn = property(_get_connection)
-
-    def nova_dhcp_filter(self):
-        """The standard allow-dhcp-server filter is an <ip> one, so it uses
-           ebtables to allow traffic through. Without a corresponding rule in
-           iptables, it'll get blocked anyway."""
-
-        return '''<filter name='nova-allow-dhcp-server' chain='ipv4'>
-                    <uuid>891e4787-e5c0-d59b-cbd6-41bc3c6b36fc</uuid>
-                    <rule action='accept' direction='out'
-                          priority='100'>
-                      <udp srcipaddr='0.0.0.0'
-                           dstipaddr='255.255.255.255'
-                           srcportstart='68'
-                           dstportstart='67'/>
-                    </rule>
-                    <rule action='accept' direction='in'
-                          priority='100'>
-                      <udp srcipaddr='$DHCPSERVER'
-                           srcportstart='67'
-                           dstportstart='68'/>
-                    </rule>
-                  </filter>'''
-
-    def nova_ra_filter(self):
-        return '''<filter name='nova-allow-ra-server' chain='root'>
-                            <uuid>d707fa71-4fb5-4b27-9ab7-ba5ca19c8804</uuid>
-                              <rule action='accept' direction='inout'
-                                    priority='100'>
-                                <icmpv6 srcipaddr='$RASERVER'/>
-                              </rule>
-                            </filter>'''
-
-    def setup_basic_filtering(self, instance, network_info=None):
-        """Set up basic filtering (MAC, IP, and ARP spoofing protection)"""
-        logging.info('called setup_basic_filtering in nwfilter')
-
-        if not network_info:
-            network_info = _get_network_info(instance)
-
-        if self.handle_security_groups:
-            # No point in setting up a filter set that we'll be overriding
-            # anyway.
-            return
-
-        logging.info('ensuring static filters')
-        self._ensure_static_filters()
-
-        if instance['image_id'] == str(FLAGS.vpn_image_id):
-            base_filter = 'nova-vpn'
-        else:
-            base_filter = 'nova-base'
-
-        for (network, mapping) in network_info:
-            nic_id = mapping['mac'].replace(':', '')
-            instance_filter_name = self._instance_filter_name(instance, nic_id)
-            self._define_filter(self._filter_container(instance_filter_name,
-                                                       [base_filter]))
-
-    def _ensure_static_filters(self):
-        if self.static_filters_configured:
-            return
-
-        self._define_filter(self._filter_container('nova-base',
-                                                   ['no-mac-spoofing',
-                                                    'no-ip-spoofing',
-                                                    'no-arp-spoofing',
-                                                    'allow-dhcp-server']))
-        self._define_filter(self._filter_container('nova-vpn',
-                                                   ['allow-dhcp-server']))
-        self._define_filter(self.nova_base_ipv4_filter)
-        self._define_filter(self.nova_base_ipv6_filter)
-        self._define_filter(self.nova_dhcp_filter)
-        self._define_filter(self.nova_ra_filter)
-        if FLAGS.allow_project_net_traffic:
-            self._define_filter(self.nova_project_filter)
-            if FLAGS.use_ipv6:
-                self._define_filter(self.nova_project_filter_v6)
-
-        self.static_filters_configured = True
-
-    def _filter_container(self, name, filters):
-        xml = '''<filter name='%s' chain='root'>%s</filter>''' % (
-                 name,
-                 ''.join(["<filterref filter='%s'/>" % (f,) for f in filters]))
-        return xml
-
-    def nova_base_ipv4_filter(self):
-        retval = "<filter name='nova-base-ipv4' chain='ipv4'>"
-        for protocol in ['tcp', 'udp', 'icmp']:
-            for direction, action, priority in [('out', 'accept', 399),
-                                                ('in', 'drop', 400)]:
-                retval += """<rule action='%s' direction='%s' priority='%d'>
-                               <%s />
-                             </rule>""" % (action, direction,
-                                              priority, protocol)
-        retval += '</filter>'
-        return retval
-
-    def nova_base_ipv6_filter(self):
-        retval = "<filter name='nova-base-ipv6' chain='ipv6'>"
-        for protocol in ['tcp-ipv6', 'udp-ipv6', 'icmpv6']:
-            for direction, action, priority in [('out', 'accept', 399),
-                                                ('in', 'drop', 400)]:
-                retval += """<rule action='%s' direction='%s' priority='%d'>
-                               <%s />
-                             </rule>""" % (action, direction,
-                                              priority, protocol)
-        retval += '</filter>'
-        return retval
-
-    def nova_project_filter(self):
-        retval = "<filter name='nova-project' chain='ipv4'>"
-        for protocol in ['tcp', 'udp', 'icmp']:
-            retval += """<rule action='accept' direction='in' priority='200'>
-                           <%s srcipaddr='$PROJNET' srcipmask='$PROJMASK' />
-                         </rule>""" % protocol
-        retval += '</filter>'
-        return retval
-
-    def nova_project_filter_v6(self):
-        retval = "<filter name='nova-project-v6' chain='ipv6'>"
-        for protocol in ['tcp-ipv6', 'udp-ipv6', 'icmpv6']:
-            retval += """<rule action='accept' direction='inout'
-                                                   priority='200'>
-                           <%s srcipaddr='$PROJNETV6'
-                               srcipmask='$PROJMASKV6' />
-                         </rule>""" % (protocol)
-        retval += '</filter>'
-        return retval
-
-    def _define_filter(self, xml):
-        if callable(xml):
-            xml = xml()
-        # execute in a native thread and block current greenthread until done
-        tpool.execute(self._conn.nwfilterDefineXML, xml)
-
-    def unfilter_instance(self, instance):
-        # Nothing to do
-        pass
-
-    def prepare_instance_filter(self, instance, network_info=None):
-        """
-        Creates an NWFilter for the given instance. In the process,
-        it makes sure the filters for the security groups as well as
-        the base filter are all in place.
-        """
-        if not network_info:
-            network_info = _get_network_info(instance)
-
-        ctxt = context.get_admin_context()
-
-        instance_secgroup_filter_name = \
-            '%s-secgroup' % (self._instance_filter_name(instance))
-            #% (instance_filter_name,)
-
-        instance_secgroup_filter_children = ['nova-base-ipv4',
-                                             'nova-base-ipv6',
-                                             'nova-allow-dhcp-server']
-
-        if FLAGS.use_ipv6:
-            networks = [network for (network, _m) in network_info if
-                        network['gateway_v6']]
-
-            if networks:
-                instance_secgroup_filter_children.\
-                    append('nova-allow-ra-server')
-
-        for security_group in \
-                db.security_group_get_by_instance(ctxt, instance['id']):
-
-            self.refresh_security_group_rules(security_group['id'])
-
-            instance_secgroup_filter_children.append('nova-secgroup-%s' %
-                                                    security_group['id'])
-
-            self._define_filter(
-                    self._filter_container(instance_secgroup_filter_name,
-                                           instance_secgroup_filter_children))
-
-        network_filters = self.\
-            _create_network_filters(instance, network_info,
-                                    instance_secgroup_filter_name)
-
-        for (name, children) in network_filters:
-            self._define_filters(name, children)
-
-    def _create_network_filters(self, instance, network_info,
-                               instance_secgroup_filter_name):
-        if instance['image_id'] == str(FLAGS.vpn_image_id):
-            base_filter = 'nova-vpn'
-        else:
-            base_filter = 'nova-base'
-
-        result = []
-        for (_n, mapping) in network_info:
-            nic_id = mapping['mac'].replace(':', '')
-            instance_filter_name = self._instance_filter_name(instance, nic_id)
-            instance_filter_children = [base_filter,
-                                        instance_secgroup_filter_name]
-
-            if FLAGS.allow_project_net_traffic:
-                instance_filter_children.append('nova-project')
-                if FLAGS.use_ipv6:
-                    instance_filter_children.append('nova-project-v6')
-
-            result.append((instance_filter_name, instance_filter_children))
-
-        return result
-
-    def _define_filters(self, filter_name, filter_children):
-        self._define_filter(self._filter_container(filter_name,
-                                                   filter_children))
-
-    def refresh_security_group_rules(self,
-                                     security_group_id,
-                                     network_info=None):
-        return self._define_filter(
-                   self.security_group_to_nwfilter_xml(security_group_id))
-
-    def security_group_to_nwfilter_xml(self, security_group_id):
-        security_group = db.security_group_get(context.get_admin_context(),
-                                               security_group_id)
-        rule_xml = ""
-        v6protocol = {'tcp': 'tcp-ipv6', 'udp': 'udp-ipv6', 'icmp': 'icmpv6'}
-        for rule in security_group.rules:
-            rule_xml += "<rule action='accept' direction='in' priority='300'>"
-            if rule.cidr:
-                version = _get_ip_version(rule.cidr)
-                if(FLAGS.use_ipv6 and version == 6):
-                    net, prefixlen = _get_net_and_prefixlen(rule.cidr)
-                    rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \
-                                (v6protocol[rule.protocol], net, prefixlen)
-                else:
-                    net, mask = _get_net_and_mask(rule.cidr)
-                    rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \
-                                (rule.protocol, net, mask)
-                if rule.protocol in ['tcp', 'udp']:
-                    rule_xml += "dstportstart='%s' dstportend='%s' " % \
-                                (rule.from_port, rule.to_port)
-                elif rule.protocol == 'icmp':
-                    LOG.info('rule.protocol: %r, rule.from_port: %r, '
-                             'rule.to_port: %r', rule.protocol,
-                             rule.from_port, rule.to_port)
-                    if rule.from_port != -1:
-                        rule_xml += "type='%s' " % rule.from_port
-                    if rule.to_port != -1:
-                        rule_xml += "code='%s' " % rule.to_port
-
-                rule_xml += '/>\n'
-            rule_xml += "</rule>\n"
-        xml = "<filter name='nova-secgroup-%s' " % security_group_id
-        if(FLAGS.use_ipv6):
-            xml += "chain='root'>%s</filter>" % rule_xml
-        else:
-            xml += "chain='ipv4'>%s</filter>" % rule_xml
-        return xml
-
-    def _instance_filter_name(self, instance, nic_id=None):
-        if not nic_id:
-            return 'nova-instance-%s' % (instance['name'])
-        return 'nova-instance-%s-%s' % (instance['name'], nic_id)
-
-    def instance_filter_exists(self, instance):
-        """Check nova-instance-instance-xxx exists"""
-        network_info = _get_network_info(instance)
-        for (network, mapping) in network_info:
-            nic_id = mapping['mac'].replace(':', '')
-            instance_filter_name = self._instance_filter_name(instance, nic_id)
-            try:
-                self._conn.nwfilterLookupByName(instance_filter_name)
-            except libvirt.libvirtError:
-                name = instance.name
-                LOG.debug(_('The nwfilter(%(instance_filter_name)s) for'
-                            '%(name)s is not found.') % locals())
-                return False
-        return True
-
-
-class IptablesFirewallDriver(FirewallDriver):
-    def __init__(self, execute=None, **kwargs):
-        from nova.network import linux_net
-        self.iptables = linux_net.iptables_manager
-        self.instances = {}
-        self.nwfilter = NWFilterFirewall(kwargs['get_connection'])
-
-        self.iptables.ipv4['filter'].add_chain('sg-fallback')
-        self.iptables.ipv4['filter'].add_rule('sg-fallback', '-j DROP')
-        self.iptables.ipv6['filter'].add_chain('sg-fallback')
-        self.iptables.ipv6['filter'].add_rule('sg-fallback', '-j DROP')
-
-    def setup_basic_filtering(self, instance, network_info=None):
-        """Use NWFilter from libvirt for this."""
-        if not network_info:
-            network_info = _get_network_info(instance)
-        return self.nwfilter.setup_basic_filtering(instance, network_info)
-
-    def apply_instance_filter(self, instance):
-        """No-op. Everything is done in prepare_instance_filter"""
-        pass
-
-    def unfilter_instance(self, instance):
-        if self.instances.pop(instance['id'], None):
-            self.remove_filters_for_instance(instance)
-            self.iptables.apply()
-        else:
-            LOG.info(_('Attempted to unfilter instance %s which is not '
-                     'filtered'), instance['id'])
-
-    def prepare_instance_filter(self, instance, network_info=None):
-        if not network_info:
-            network_info = _get_network_info(instance)
-        self.instances[instance['id']] = instance
-        self.add_filters_for_instance(instance, network_info)
-        self.iptables.apply()
-
-    def _create_filter(self, ips, chain_name):
-        return ['-d %s -j $%s' % (ip, chain_name) for ip in ips]
-
-    def _filters_for_instance(self, chain_name, network_info):
-        ips_v4 = [ip['ip'] for (_n, mapping) in network_info
-                 for ip in mapping['ips']]
-        ipv4_rules = self._create_filter(ips_v4, chain_name)
-
-        ipv6_rules = []
-        if FLAGS.use_ipv6:
-            ips_v6 = [ip['ip'] for (_n, mapping) in network_info
-                     for ip in mapping['ip6s']]
-            ipv6_rules = self._create_filter(ips_v6, chain_name)
-
-        return ipv4_rules, ipv6_rules
-
-    def _add_filters(self, chain_name, ipv4_rules, ipv6_rules):
-        for rule in ipv4_rules:
-            self.iptables.ipv4['filter'].add_rule(chain_name, rule)
-
-        if FLAGS.use_ipv6:
-            for rule in ipv6_rules:
-                self.iptables.ipv6['filter'].add_rule(chain_name, rule)
-
-    def add_filters_for_instance(self, instance, network_info=None):
-        chain_name = self._instance_chain_name(instance)
-        if FLAGS.use_ipv6:
-            self.iptables.ipv6['filter'].add_chain(chain_name)
-        self.iptables.ipv4['filter'].add_chain(chain_name)
-        ipv4_rules, ipv6_rules = self._filters_for_instance(chain_name,
-                                                            network_info)
-        self._add_filters('local', ipv4_rules, ipv6_rules)
-        ipv4_rules, ipv6_rules = self.instance_rules(instance, network_info)
-        self._add_filters(chain_name, ipv4_rules, ipv6_rules)
-
-    def remove_filters_for_instance(self, instance):
-        chain_name = self._instance_chain_name(instance)
-
-        self.iptables.ipv4['filter'].remove_chain(chain_name)
-        if FLAGS.use_ipv6:
-            self.iptables.ipv6['filter'].remove_chain(chain_name)
-
-    def instance_rules(self, instance, network_info=None):
-        if not network_info:
-            network_info = _get_network_info(instance)
-        ctxt = context.get_admin_context()
-
-        ipv4_rules = []
-        ipv6_rules = []
-
-        # Always drop invalid packets
-        ipv4_rules += ['-m state --state ' 'INVALID -j DROP']
-        ipv6_rules += ['-m state --state ' 'INVALID -j DROP']
-
-        # Allow established connections
-        ipv4_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT']
-        ipv6_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT']
-
-        dhcp_servers = [network['gateway'] for (network, _m) in network_info]
-
-        for dhcp_server in dhcp_servers:
-            ipv4_rules.append('-s %s -p udp --sport 67 --dport 68 '
-                              '-j ACCEPT' % (dhcp_server,))
-
-        #Allow project network traffic
-        if FLAGS.allow_project_net_traffic:
-            cidrs = [network['cidr'] for (network, _m) in network_info]
-            for cidr in cidrs:
-                ipv4_rules.append('-s %s -j ACCEPT' % (cidr,))
-
-        # We wrap these in FLAGS.use_ipv6 because they might cause
-        # a DB lookup. The other ones are just list operations, so
-        # they're not worth the clutter.
-        if FLAGS.use_ipv6:
-            # Allow RA responses
-            gateways_v6 = [network['gateway_v6'] for (network, _) in
-                           network_info]
-            for gateway_v6 in gateways_v6:
-                ipv6_rules.append(
-                        '-s %s/128 -p icmpv6 -j ACCEPT' % (gateway_v6,))
-
-            #Allow project network traffic
-            if FLAGS.allow_project_net_traffic:
-                cidrv6s = [network['cidr_v6'] for (network, _m)
-                          in network_info]
-
-                for cidrv6 in cidrv6s:
-                    ipv6_rules.append('-s %s -j ACCEPT' % (cidrv6,))
-
-        security_groups = db.security_group_get_by_instance(ctxt,
-                                                            instance['id'])
-
-        # then, security group chains and rules
-        for security_group in security_groups:
-            rules = db.security_group_rule_get_by_security_group(ctxt,
-                                                          security_group['id'])
-
-            for rule in rules:
-                logging.info('%r', rule)
-
-                if not rule.cidr:
-                    # Eventually, a mechanism to grant access for security
-                    # groups will turn up here. It'll use ipsets.
-                    continue
-
-                version = _get_ip_version(rule.cidr)
-                if version == 4:
-                    rules = ipv4_rules
-                else:
-                    rules = ipv6_rules
-
-                protocol = rule.protocol
-                if version == 6 and rule.protocol == 'icmp':
-                    protocol = 'icmpv6'
-
-                args = ['-p', protocol, '-s', rule.cidr]
-
-                if rule.protocol in ['udp', 'tcp']:
-                    if rule.from_port == rule.to_port:
-                        args += ['--dport', '%s' % (rule.from_port,)]
-                    else:
-                        args += ['-m', 'multiport',
-                                 '--dports', '%s:%s' % (rule.from_port,
-                                                        rule.to_port)]
-                elif rule.protocol == 'icmp':
-                    icmp_type = rule.from_port
-                    icmp_code = rule.to_port
-
-                    if icmp_type == -1:
-                        icmp_type_arg = None
-                    else:
-                        icmp_type_arg = '%s' % icmp_type
-                        if not icmp_code == -1:
-                            icmp_type_arg += '/%s' % icmp_code
-
-                    if icmp_type_arg:
-                        if version == 4:
-                            args += ['-m', 'icmp', '--icmp-type',
-                                     icmp_type_arg]
-                        elif version == 6:
-                            args += ['-m', 'icmp6', '--icmpv6-type',
-                                     icmp_type_arg]
-
-                args += ['-j ACCEPT']
-                rules += [' '.join(args)]
-
-        ipv4_rules += ['-j $sg-fallback']
-        ipv6_rules += ['-j $sg-fallback']
-
-        return ipv4_rules, ipv6_rules
-
-    def instance_filter_exists(self, instance):
-        """Check nova-instance-instance-xxx exists"""
-        return self.nwfilter.instance_filter_exists(instance)
-
-    def refresh_security_group_members(self, security_group):
-        pass
-
-    def refresh_security_group_rules(self, security_group, network_info=None):
-        self.do_refresh_security_group_rules(security_group, network_info)
-        self.iptables.apply()
-
-    @utils.synchronized('iptables', external=True)
-    def do_refresh_security_group_rules(self,
-                                        security_group,
-                                        network_info=None):
-        for instance in self.instances.values():
-            self.remove_filters_for_instance(instance)
-            if not network_info:
-                network_info = _get_network_info(instance)
-            self.add_filters_for_instance(instance, network_info)
-
-    def _security_group_chain_name(self, security_group_id):
-        return 'nova-sg-%s' % (security_group_id,)
-
-    def _instance_chain_name(self, instance):
-        return 'inst-%s' % (instance['id'],)
diff --git a/nova/virt/libvirt/firewall.py b/nova/virt/libvirt/firewall.py
new file mode 100644
index 000000000..b99f2ffb0
--- /dev/null
+++ b/nova/virt/libvirt/firewall.py
@@ -0,0 +1,820 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+# Copyright (c) 2010 Citrix Systems, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+from eventlet import tpool
+
+from nova import context
+from nova import db
+from nova import flags
+from nova import log as logging
+from nova import utils
+from nova.virt.libvirt import netutils
+
+
+LOG = logging.getLogger("nova.virt.libvirt.firewall")
+FLAGS = flags.FLAGS
+
+
+try:
+    import libvirt
+except ImportError:
+    LOG.warn(_("Libvirt module could not be loaded. NWFilterFirewall will "
+               "not work correctly."))
+
+
+class FirewallDriver(object):
+    def prepare_instance_filter(self, instance, network_info=None):
+        """Prepare filters for the instance.
+
+        At this point, the instance isn't running yet."""
+        raise NotImplementedError()
+
+    def unfilter_instance(self, instance):
+        """Stop filtering instance"""
+        raise NotImplementedError()
+
+    def apply_instance_filter(self, instance):
+        """Apply instance filter.
+
+        Once this method returns, the instance should be firewalled
+        appropriately. This method should as far as possible be a
+        no-op. It's vastly preferred to get everything set up in
+        prepare_instance_filter.
+        """
+        raise NotImplementedError()
+
+    def refresh_security_group_rules(self,
+                                     security_group_id,
+                                     network_info=None):
+        """Refresh security group rules from data store
+
+        Gets called when a rule has been added to or removed from
+        the security group."""
+        raise NotImplementedError()
+
+    def refresh_security_group_members(self, security_group_id):
+        """Refresh security group members from data store
+
+        Gets called when an instance gets added to or removed from
+        the security group."""
+        raise NotImplementedError()
+
+    def refresh_provider_fw_rules(self):
+        """Refresh common rules for all hosts/instances from data store.
+
+        Gets called when a rule has been added to or removed from
+        the list of rules (via admin api).
+
+        """
+        raise NotImplementedError()
+
+    def setup_basic_filtering(self, instance, network_info=None):
+        """Create rules to block spoofing and allow dhcp.
+
+        This gets called when spawning an instance, before
+        :method:`prepare_instance_filter`.
+
+        """
+        raise NotImplementedError()
+
+    def instance_filter_exists(self, instance):
+        """Check nova-instance-instance-xxx exists"""
+        raise NotImplementedError()
+
+
+class NWFilterFirewall(FirewallDriver):
+    """
+    This class implements a network filtering mechanism versatile
+    enough for EC2 style Security Group filtering by leveraging
+    libvirt's nwfilter.
+
+    First, all instances get a filter ("nova-base-filter") applied.
+    This filter provides some basic security such as protection against
+    MAC spoofing, IP spoofing, and ARP spoofing.
+
+    This filter drops all incoming ipv4 and ipv6 connections.
+    Outgoing connections are never blocked.
+
+    Second, every security group maps to a nwfilter filter(*).
+    NWFilters can be updated at runtime and changes are applied
+    immediately, so changes to security groups can be applied at
+    runtime (as mandated by the spec).
+
+    Security group rules are named "nova-secgroup-<id>" where <id>
+    is the internal id of the security group. They're applied only on
+    hosts that have instances in the security group in question.
+
+    Updates to security groups are done by updating the data model
+    (in response to API calls) followed by a request sent to all
+    the nodes with instances in the security group to refresh the
+    security group.
+
+    Each instance has its own NWFilter, which references the above
+    mentioned security group NWFilters. This was done because
+    interfaces can only reference one filter while filters can
+    reference multiple other filters. This has the added benefit of
+    actually being able to add and remove security groups from an
+    instance at run time. This functionality is not exposed anywhere,
+    though.
+
+    Outstanding questions:
+
+    The name is unique, so would there be any good reason to sync
+    the uuid across the nodes (by assigning it from the datamodel)?
+
+
+    (*) This sentence brought to you by the redundancy department of
+        redundancy.
+
+    """
+
+    def __init__(self, get_connection, **kwargs):
+        self._libvirt_get_connection = get_connection
+        self.static_filters_configured = False
+        self.handle_security_groups = False
+
+    def apply_instance_filter(self, instance):
+        """No-op. Everything is done in prepare_instance_filter"""
+        pass
+
+    def _get_connection(self):
+        return self._libvirt_get_connection()
+    _conn = property(_get_connection)
+
+    def nova_dhcp_filter(self):
+        """The standard allow-dhcp-server filter is an <ip> one, so it uses
+           ebtables to allow traffic through. Without a corresponding rule in
+           iptables, it'll get blocked anyway."""
+
+        return '''<filter name='nova-allow-dhcp-server' chain='ipv4'>
+                    <uuid>891e4787-e5c0-d59b-cbd6-41bc3c6b36fc</uuid>
+                    <rule action='accept' direction='out'
+                          priority='100'>
+                      <udp srcipaddr='0.0.0.0'
+                           dstipaddr='255.255.255.255'
+                           srcportstart='68'
+                           dstportstart='67'/>
+                    </rule>
+                    <rule action='accept' direction='in'
+                          priority='100'>
+                      <udp srcipaddr='$DHCPSERVER'
+                           srcportstart='67'
+                           dstportstart='68'/>
+                    </rule>
+                  </filter>'''
+
+    def nova_ra_filter(self):
+        return '''<filter name='nova-allow-ra-server' chain='root'>
+                            <uuid>d707fa71-4fb5-4b27-9ab7-ba5ca19c8804</uuid>
+                              <rule action='accept' direction='inout'
+                                    priority='100'>
+                                <icmpv6 srcipaddr='$RASERVER'/>
+                              </rule>
+                            </filter>'''
+
+    def setup_basic_filtering(self, instance, network_info=None):
+        """Set up basic filtering (MAC, IP, and ARP spoofing protection)"""
+        logging.info('called setup_basic_filtering in nwfilter')
+
+        if not network_info:
+            network_info = netutils.get_network_info(instance)
+
+        if self.handle_security_groups:
+            # No point in setting up a filter set that we'll be overriding
+            # anyway.
+            return
+
+        logging.info('ensuring static filters')
+        self._ensure_static_filters()
+
+        if instance['image_ref'] == str(FLAGS.vpn_image_id):
+            base_filter = 'nova-vpn'
+        else:
+            base_filter = 'nova-base'
+
+        for (network, mapping) in network_info:
+            nic_id = mapping['mac'].replace(':', '')
+            instance_filter_name = self._instance_filter_name(instance, nic_id)
+            self._define_filter(self._filter_container(instance_filter_name,
+                                                       [base_filter]))
+
+    def _ensure_static_filters(self):
+        """Static filters are filters that have no need to be IP aware.
+
+        There is no configuration or tuneability of these filters, so they
+        can be set up once and forgotten about.
+
+        """
+
+        if self.static_filters_configured:
+            return
+
+        self._define_filter(self._filter_container('nova-base',
+                                                   ['no-mac-spoofing',
+                                                    'no-ip-spoofing',
+                                                    'no-arp-spoofing',
+                                                    'allow-dhcp-server']))
+        self._define_filter(self._filter_container('nova-vpn',
+                                                   ['allow-dhcp-server']))
+        self._define_filter(self.nova_base_ipv4_filter)
+        self._define_filter(self.nova_base_ipv6_filter)
+        self._define_filter(self.nova_dhcp_filter)
+        self._define_filter(self.nova_ra_filter)
+        if FLAGS.allow_project_net_traffic:
+            self._define_filter(self.nova_project_filter)
+            if FLAGS.use_ipv6:
+                self._define_filter(self.nova_project_filter_v6)
+
+        self.static_filters_configured = True
+
+    def _filter_container(self, name, filters):
+        xml = '''<filter name='%s' chain='root'>%s</filter>''' % (
+                 name,
+                 ''.join(["<filterref filter='%s'/>" % (f,) for f in filters]))
+        return xml
+
+    def nova_base_ipv4_filter(self):
+        retval = "<filter name='nova-base-ipv4' chain='ipv4'>"
+        for protocol in ['tcp', 'udp', 'icmp']:
+            for direction, action, priority in [('out', 'accept', 399),
+                                                ('in', 'drop', 400)]:
+                retval += """<rule action='%s' direction='%s' priority='%d'>
+                               <%s />
+                             </rule>""" % (action, direction,
+                                              priority, protocol)
+        retval += '</filter>'
+        return retval
+
+    def nova_base_ipv6_filter(self):
+        retval = "<filter name='nova-base-ipv6' chain='ipv6'>"
+        for protocol in ['tcp-ipv6', 'udp-ipv6', 'icmpv6']:
+            for direction, action, priority in [('out', 'accept', 399),
+                                                ('in', 'drop', 400)]:
+                retval += """<rule action='%s' direction='%s' priority='%d'>
+                               <%s />
+                             </rule>""" % (action, direction,
+                                              priority, protocol)
+        retval += '</filter>'
+        return retval
+
+    def nova_project_filter(self):
+        retval = "<filter name='nova-project' chain='ipv4'>"
+        for protocol in ['tcp', 'udp', 'icmp']:
+            retval += """<rule action='accept' direction='in' priority='200'>
+                           <%s srcipaddr='$PROJNET' srcipmask='$PROJMASK' />
+                         </rule>""" % protocol
+        retval += '</filter>'
+        return retval
+
+    def nova_project_filter_v6(self):
+        retval = "<filter name='nova-project-v6' chain='ipv6'>"
+        for protocol in ['tcp-ipv6', 'udp-ipv6', 'icmpv6']:
+            retval += """<rule action='accept' direction='inout'
+                                                   priority='200'>
+                           <%s srcipaddr='$PROJNETV6'
+                               srcipmask='$PROJMASKV6' />
+                         </rule>""" % (protocol)
+        retval += '</filter>'
+        return retval
+
+    def _define_filter(self, xml):
+        if callable(xml):
+            xml = xml()
+        # execute in a native thread and block current greenthread until done
+        tpool.execute(self._conn.nwfilterDefineXML, xml)
+
+    def unfilter_instance(self, instance):
+        """Clear out the nwfilter rules."""
+        network_info = netutils.get_network_info(instance)
+        instance_name = instance.name
+        for (network, mapping) in network_info:
+            nic_id = mapping['mac'].replace(':', '')
+            instance_filter_name = self._instance_filter_name(instance, nic_id)
+
+            try:
+                self._conn.nwfilterLookupByName(instance_filter_name).\
+                                                    undefine()
+            except libvirt.libvirtError:
+                LOG.debug(_('The nwfilter(%(instance_filter_name)s) '
+                            'for %(instance_name)s is not found.') % locals())
+
+        instance_secgroup_filter_name =\
+            '%s-secgroup' % (self._instance_filter_name(instance))
+
+        try:
+            self._conn.nwfilterLookupByName(instance_secgroup_filter_name)\
+                                            .undefine()
+        except libvirt.libvirtError:
+            LOG.debug(_('The nwfilter(%(instance_secgroup_filter_name)s) '
+                        'for %(instance_name)s is not found.') % locals())
+
+    def prepare_instance_filter(self, instance, network_info=None):
+        """Creates an NWFilter for the given instance.
+
+        In the process, it makes sure the filters for the provider blocks,
+        security groups, and base filter are all in place.
+
+        """
+        if not network_info:
+            network_info = netutils.get_network_info(instance)
+
+        self.refresh_provider_fw_rules()
+
+        ctxt = context.get_admin_context()
+
+        instance_secgroup_filter_name = \
+            '%s-secgroup' % (self._instance_filter_name(instance))
+
+        instance_secgroup_filter_children = ['nova-base-ipv4',
+                                             'nova-base-ipv6',
+                                             'nova-allow-dhcp-server']
+
+        if FLAGS.use_ipv6:
+            networks = [network for (network, _m) in network_info if
+                        network['gateway_v6']]
+
+            if networks:
+                instance_secgroup_filter_children.\
+                    append('nova-allow-ra-server')
+
+        for security_group in \
+                db.security_group_get_by_instance(ctxt, instance['id']):
+
+            self.refresh_security_group_rules(security_group['id'])
+
+            instance_secgroup_filter_children.append('nova-secgroup-%s' %
+                                                    security_group['id'])
+
+            self._define_filter(
+                    self._filter_container(instance_secgroup_filter_name,
+                                           instance_secgroup_filter_children))
+
+        network_filters = self.\
+            _create_network_filters(instance, network_info,
+                                    instance_secgroup_filter_name)
+
+        for (name, children) in network_filters:
+            self._define_filters(name, children)
+
+    def _create_network_filters(self, instance, network_info,
+                               instance_secgroup_filter_name):
+        if instance['image_ref'] == str(FLAGS.vpn_image_id):
+            base_filter = 'nova-vpn'
+        else:
+            base_filter = 'nova-base'
+
+        result = []
+        for (_n, mapping) in network_info:
+            nic_id = mapping['mac'].replace(':', '')
+            instance_filter_name = self._instance_filter_name(instance, nic_id)
+            instance_filter_children = [base_filter, 'nova-provider-rules',
+                                        instance_secgroup_filter_name]
+
+            if FLAGS.allow_project_net_traffic:
+                instance_filter_children.append('nova-project')
+                if FLAGS.use_ipv6:
+                    instance_filter_children.append('nova-project-v6')
+
+            result.append((instance_filter_name, instance_filter_children))
+
+        return result
+
+    def _define_filters(self, filter_name, filter_children):
+        self._define_filter(self._filter_container(filter_name,
+                                                   filter_children))
+
+    def refresh_security_group_rules(self,
+                                     security_group_id,
+                                     network_info=None):
+        return self._define_filter(
+                   self.security_group_to_nwfilter_xml(security_group_id))
+
+    def refresh_provider_fw_rules(self):
+        """Update rules for all instances.
+
+        This is part of the FirewallDriver API and is called when the
+        provider firewall rules change in the database.  In the
+        `prepare_instance_filter` we add a reference to the
+        'nova-provider-rules' filter for each instance's firewall, and
+        by changing that filter we update them all.
+
+        """
+        xml = self.provider_fw_to_nwfilter_xml()
+        return self._define_filter(xml)
+
+    def security_group_to_nwfilter_xml(self, security_group_id):
+        security_group = db.security_group_get(context.get_admin_context(),
+                                               security_group_id)
+        rule_xml = ""
+        v6protocol = {'tcp': 'tcp-ipv6', 'udp': 'udp-ipv6', 'icmp': 'icmpv6'}
+        for rule in security_group.rules:
+            rule_xml += "<rule action='accept' direction='in' priority='300'>"
+            if rule.cidr:
+                version = netutils.get_ip_version(rule.cidr)
+                if(FLAGS.use_ipv6 and version == 6):
+                    net, prefixlen = netutils.get_net_and_prefixlen(rule.cidr)
+                    rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \
+                                (v6protocol[rule.protocol], net, prefixlen)
+                else:
+                    net, mask = netutils.get_net_and_mask(rule.cidr)
+                    rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \
+                                (rule.protocol, net, mask)
+                if rule.protocol in ['tcp', 'udp']:
+                    rule_xml += "dstportstart='%s' dstportend='%s' " % \
+                                (rule.from_port, rule.to_port)
+                elif rule.protocol == 'icmp':
+                    LOG.info('rule.protocol: %r, rule.from_port: %r, '
+                             'rule.to_port: %r', rule.protocol,
+                             rule.from_port, rule.to_port)
+                    if rule.from_port != -1:
+                        rule_xml += "type='%s' " % rule.from_port
+                    if rule.to_port != -1:
+                        rule_xml += "code='%s' " % rule.to_port
+
+                rule_xml += '/>\n'
+            rule_xml += "</rule>\n"
+        xml = "<filter name='nova-secgroup-%s' " % security_group_id
+        if(FLAGS.use_ipv6):
+            xml += "chain='root'>%s</filter>" % rule_xml
+        else:
+            xml += "chain='ipv4'>%s</filter>" % rule_xml
+        return xml
+
+    def provider_fw_to_nwfilter_xml(self):
+        """Compose a filter of drop rules from specified cidrs."""
+        rule_xml = ""
+        v6protocol = {'tcp': 'tcp-ipv6', 'udp': 'udp-ipv6', 'icmp': 'icmpv6'}
+        rules = db.provider_fw_rule_get_all(context.get_admin_context())
+        for rule in rules:
+            rule_xml += "<rule action='block' direction='in' priority='150'>"
+            version = netutils.get_ip_version(rule.cidr)
+            if(FLAGS.use_ipv6 and version == 6):
+                net, prefixlen = netutils.get_net_and_prefixlen(rule.cidr)
+                rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \
+                            (v6protocol[rule.protocol], net, prefixlen)
+            else:
+                net, mask = netutils.get_net_and_mask(rule.cidr)
+                rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \
+                            (rule.protocol, net, mask)
+            if rule.protocol in ['tcp', 'udp']:
+                rule_xml += "dstportstart='%s' dstportend='%s' " % \
+                            (rule.from_port, rule.to_port)
+            elif rule.protocol == 'icmp':
+                LOG.info('rule.protocol: %r, rule.from_port: %r, '
+                         'rule.to_port: %r', rule.protocol,
+                         rule.from_port, rule.to_port)
+                if rule.from_port != -1:
+                    rule_xml += "type='%s' " % rule.from_port
+                if rule.to_port != -1:
+                    rule_xml += "code='%s' " % rule.to_port
+
+                rule_xml += '/>\n'
+            rule_xml += "</rule>\n"
+        xml = "<filter name='nova-provider-rules' "
+        if(FLAGS.use_ipv6):
+            xml += "chain='root'>%s</filter>" % rule_xml
+        else:
+            xml += "chain='ipv4'>%s</filter>" % rule_xml
+        return xml
+
+    def _instance_filter_name(self, instance, nic_id=None):
+        if not nic_id:
+            return 'nova-instance-%s' % (instance['name'])
+        return 'nova-instance-%s-%s' % (instance['name'], nic_id)
+
+    def instance_filter_exists(self, instance):
+        """Check nova-instance-instance-xxx exists"""
+        network_info = netutils.get_network_info(instance)
+        for (network, mapping) in network_info:
+            nic_id = mapping['mac'].replace(':', '')
+            instance_filter_name = self._instance_filter_name(instance, nic_id)
+            try:
+                self._conn.nwfilterLookupByName(instance_filter_name)
+            except libvirt.libvirtError:
+                name = instance.name
+                LOG.debug(_('The nwfilter(%(instance_filter_name)s) for'
+                            '%(name)s is not found.') % locals())
+                return False
+        return True
+
+
+class IptablesFirewallDriver(FirewallDriver):
+    def __init__(self, execute=None, **kwargs):
+        from nova.network import linux_net
+        self.iptables = linux_net.iptables_manager
+        self.instances = {}
+        self.nwfilter = NWFilterFirewall(kwargs['get_connection'])
+        self.basicly_filtered = False
+
+        self.iptables.ipv4['filter'].add_chain('sg-fallback')
+        self.iptables.ipv4['filter'].add_rule('sg-fallback', '-j DROP')
+        self.iptables.ipv6['filter'].add_chain('sg-fallback')
+        self.iptables.ipv6['filter'].add_rule('sg-fallback', '-j DROP')
+
+    def setup_basic_filtering(self, instance, network_info=None):
+        """Set up provider rules and basic NWFilter."""
+        if not network_info:
+            network_info = netutils.get_network_info(instance)
+        self.nwfilter.setup_basic_filtering(instance, network_info)
+        if not self.basicly_filtered:
+            LOG.debug(_('iptables firewall: Setup Basic Filtering'))
+            self.refresh_provider_fw_rules()
+            self.basicly_filtered = True
+
+    def apply_instance_filter(self, instance):
+        """No-op. Everything is done in prepare_instance_filter"""
+        pass
+
+    def unfilter_instance(self, instance):
+        if self.instances.pop(instance['id'], None):
+            self.remove_filters_for_instance(instance)
+            self.iptables.apply()
+            self.nwfilter.unfilter_instance(instance)
+        else:
+            LOG.info(_('Attempted to unfilter instance %s which is not '
+                     'filtered'), instance['id'])
+
+    def prepare_instance_filter(self, instance, network_info=None):
+        if not network_info:
+            network_info = netutils.get_network_info(instance)
+        self.instances[instance['id']] = instance
+        self.add_filters_for_instance(instance, network_info)
+        self.iptables.apply()
+
+    def _create_filter(self, ips, chain_name):
+        return ['-d %s -j $%s' % (ip, chain_name) for ip in ips]
+
+    def _filters_for_instance(self, chain_name, network_info):
+        ips_v4 = [ip['ip'] for (_n, mapping) in network_info
+                 for ip in mapping['ips']]
+        ipv4_rules = self._create_filter(ips_v4, chain_name)
+
+        ipv6_rules = []
+        if FLAGS.use_ipv6:
+            ips_v6 = [ip['ip'] for (_n, mapping) in network_info
+                     for ip in mapping['ip6s']]
+            ipv6_rules = self._create_filter(ips_v6, chain_name)
+
+        return ipv4_rules, ipv6_rules
+
+    def _add_filters(self, chain_name, ipv4_rules, ipv6_rules):
+        for rule in ipv4_rules:
+            self.iptables.ipv4['filter'].add_rule(chain_name, rule)
+
+        if FLAGS.use_ipv6:
+            for rule in ipv6_rules:
+                self.iptables.ipv6['filter'].add_rule(chain_name, rule)
+
+    def add_filters_for_instance(self, instance, network_info=None):
+        chain_name = self._instance_chain_name(instance)
+        if FLAGS.use_ipv6:
+            self.iptables.ipv6['filter'].add_chain(chain_name)
+        self.iptables.ipv4['filter'].add_chain(chain_name)
+        ipv4_rules, ipv6_rules = self._filters_for_instance(chain_name,
+                                                            network_info)
+        self._add_filters('local', ipv4_rules, ipv6_rules)
+        ipv4_rules, ipv6_rules = self.instance_rules(instance, network_info)
+        self._add_filters(chain_name, ipv4_rules, ipv6_rules)
+
+    def remove_filters_for_instance(self, instance):
+        chain_name = self._instance_chain_name(instance)
+
+        self.iptables.ipv4['filter'].remove_chain(chain_name)
+        if FLAGS.use_ipv6:
+            self.iptables.ipv6['filter'].remove_chain(chain_name)
+
+    def instance_rules(self, instance, network_info=None):
+        if not network_info:
+            network_info = netutils.get_network_info(instance)
+        ctxt = context.get_admin_context()
+
+        ipv4_rules = []
+        ipv6_rules = []
+
+        # Always drop invalid packets
+        ipv4_rules += ['-m state --state ' 'INVALID -j DROP']
+        ipv6_rules += ['-m state --state ' 'INVALID -j DROP']
+
+        # Allow established connections
+        ipv4_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT']
+        ipv6_rules += ['-m state --state ESTABLISHED,RELATED -j ACCEPT']
+
+        # Pass through provider-wide drops
+        ipv4_rules += ['-j $provider']
+        ipv6_rules += ['-j $provider']
+
+        dhcp_servers = [network['gateway'] for (network, _m) in network_info]
+
+        for dhcp_server in dhcp_servers:
+            ipv4_rules.append('-s %s -p udp --sport 67 --dport 68 '
+                              '-j ACCEPT' % (dhcp_server,))
+
+        #Allow project network traffic
+        if FLAGS.allow_project_net_traffic:
+            cidrs = [network['cidr'] for (network, _m) in network_info]
+            for cidr in cidrs:
+                ipv4_rules.append('-s %s -j ACCEPT' % (cidr,))
+
+        # We wrap these in FLAGS.use_ipv6 because they might cause
+        # a DB lookup. The other ones are just list operations, so
+        # they're not worth the clutter.
+        if FLAGS.use_ipv6:
+            # Allow RA responses
+            gateways_v6 = [network['gateway_v6'] for (network, _m) in
+                           network_info]
+            for gateway_v6 in gateways_v6:
+                ipv6_rules.append(
+                        '-s %s/128 -p icmpv6 -j ACCEPT' % (gateway_v6,))
+
+            #Allow project network traffic
+            if FLAGS.allow_project_net_traffic:
+                cidrv6s = [network['cidr_v6'] for (network, _m)
+                          in network_info]
+
+                for cidrv6 in cidrv6s:
+                    ipv6_rules.append('-s %s -j ACCEPT' % (cidrv6,))
+
+        security_groups = db.security_group_get_by_instance(ctxt,
+                                                            instance['id'])
+
+        # then, security group chains and rules
+        for security_group in security_groups:
+            rules = db.security_group_rule_get_by_security_group(ctxt,
+                                                          security_group['id'])
+
+            for rule in rules:
+                LOG.debug(_('Adding security group rule: %r'), rule)
+
+                if not rule.cidr:
+                    # Eventually, a mechanism to grant access for security
+                    # groups will turn up here. It'll use ipsets.
+                    continue
+
+                version = netutils.get_ip_version(rule.cidr)
+                if version == 4:
+                    fw_rules = ipv4_rules
+                else:
+                    fw_rules = ipv6_rules
+
+                protocol = rule.protocol
+                if version == 6 and rule.protocol == 'icmp':
+                    protocol = 'icmpv6'
+
+                args = ['-p', protocol, '-s', rule.cidr]
+
+                if rule.protocol in ['udp', 'tcp']:
+                    if rule.from_port == rule.to_port:
+                        args += ['--dport', '%s' % (rule.from_port,)]
+                    else:
+                        args += ['-m', 'multiport',
+                                 '--dports', '%s:%s' % (rule.from_port,
+                                                        rule.to_port)]
+                elif rule.protocol == 'icmp':
+                    icmp_type = rule.from_port
+                    icmp_code = rule.to_port
+
+                    if icmp_type == -1:
+                        icmp_type_arg = None
+                    else:
+                        icmp_type_arg = '%s' % icmp_type
+                        if not icmp_code == -1:
+                            icmp_type_arg += '/%s' % icmp_code
+
+                    if icmp_type_arg:
+                        if version == 4:
+                            args += ['-m', 'icmp', '--icmp-type',
+                                     icmp_type_arg]
+                        elif version == 6:
+                            args += ['-m', 'icmp6', '--icmpv6-type',
+                                     icmp_type_arg]
+
+                args += ['-j ACCEPT']
+                fw_rules += [' '.join(args)]
+
+        ipv4_rules += ['-j $sg-fallback']
+        ipv6_rules += ['-j $sg-fallback']
+
+        return ipv4_rules, ipv6_rules
+
+    def instance_filter_exists(self, instance):
+        """Check nova-instance-instance-xxx exists"""
+        return self.nwfilter.instance_filter_exists(instance)
+
+    def refresh_security_group_members(self, security_group):
+        pass
+
+    def refresh_security_group_rules(self, security_group, network_info=None):
+        self.do_refresh_security_group_rules(security_group, network_info)
+        self.iptables.apply()
+
+    @utils.synchronized('iptables', external=True)
+    def do_refresh_security_group_rules(self,
+                                        security_group,
+                                        network_info=None):
+        for instance in self.instances.values():
+            self.remove_filters_for_instance(instance)
+            if not network_info:
+                network_info = netutils.get_network_info(instance)
+            self.add_filters_for_instance(instance, network_info)
+
+    def refresh_provider_fw_rules(self):
+        """See class:FirewallDriver: docs."""
+        self._do_refresh_provider_fw_rules()
+        self.iptables.apply()
+
+    @utils.synchronized('iptables', external=True)
+    def _do_refresh_provider_fw_rules(self):
+        """Internal, synchronized version of refresh_provider_fw_rules."""
+        self._purge_provider_fw_rules()
+        self._build_provider_fw_rules()
+
+    def _purge_provider_fw_rules(self):
+        """Remove all rules from the provider chains."""
+        self.iptables.ipv4['filter'].empty_chain('provider')
+        if FLAGS.use_ipv6:
+            self.iptables.ipv6['filter'].empty_chain('provider')
+
+    def _build_provider_fw_rules(self):
+        """Create all rules for the provider IP DROPs."""
+        self.iptables.ipv4['filter'].add_chain('provider')
+        if FLAGS.use_ipv6:
+            self.iptables.ipv6['filter'].add_chain('provider')
+        ipv4_rules, ipv6_rules = self._provider_rules()
+        for rule in ipv4_rules:
+            self.iptables.ipv4['filter'].add_rule('provider', rule)
+
+        if FLAGS.use_ipv6:
+            for rule in ipv6_rules:
+                self.iptables.ipv6['filter'].add_rule('provider', rule)
+
+    def _provider_rules(self):
+        """Generate a list of rules from provider for IP4 & IP6."""
+        ctxt = context.get_admin_context()
+        ipv4_rules = []
+        ipv6_rules = []
+        rules = db.provider_fw_rule_get_all(ctxt)
+        for rule in rules:
+            LOG.debug(_('Adding provider rule: %s'), rule['cidr'])
+            version = netutils.get_ip_version(rule['cidr'])
+            if version == 4:
+                fw_rules = ipv4_rules
+            else:
+                fw_rules = ipv6_rules
+
+            protocol = rule['protocol']
+            if version == 6 and protocol == 'icmp':
+                protocol = 'icmpv6'
+
+            args = ['-p', protocol, '-s', rule['cidr']]
+
+            if protocol in ['udp', 'tcp']:
+                if rule['from_port'] == rule['to_port']:
+                    args += ['--dport', '%s' % (rule['from_port'],)]
+                else:
+                    args += ['-m', 'multiport',
+                             '--dports', '%s:%s' % (rule['from_port'],
+                                                    rule['to_port'])]
+            elif protocol == 'icmp':
+                icmp_type = rule['from_port']
+                icmp_code = rule['to_port']
+
+                if icmp_type == -1:
+                    icmp_type_arg = None
+                else:
+                    icmp_type_arg = '%s' % icmp_type
+                    if not icmp_code == -1:
+                        icmp_type_arg += '/%s' % icmp_code
+
+                if icmp_type_arg:
+                    if version == 4:
+                        args += ['-m', 'icmp', '--icmp-type',
+                                 icmp_type_arg]
+                    elif version == 6:
+                        args += ['-m', 'icmp6', '--icmpv6-type',
+                                 icmp_type_arg]
+            args += ['-j DROP']
+            fw_rules += [' '.join(args)]
+        return ipv4_rules, ipv6_rules
+
+    def _security_group_chain_name(self, security_group_id):
+        return 'nova-sg-%s' % (security_group_id,)
+
+    def _instance_chain_name(self, instance):
+        return 'inst-%s' % (instance['id'],)
diff --git a/nova/virt/libvirt/netutils.py b/nova/virt/libvirt/netutils.py
new file mode 100644
index 000000000..0bad84f7c
--- /dev/null
+++ b/nova/virt/libvirt/netutils.py
@@ -0,0 +1,97 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2010 United States Government as represented by the
+# Administrator of the National Aeronautics and Space Administration.
+# All Rights Reserved.
+# Copyright (c) 2010 Citrix Systems, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+"""Network-releated utilities for supporting libvirt connection code."""
+
+
+import netaddr
+
+from nova import context
+from nova import db
+from nova import flags
+from nova import ipv6
+from nova import utils
+
+
+FLAGS = flags.FLAGS
+
+
+def get_net_and_mask(cidr):
+    net = netaddr.IPNetwork(cidr)
+    return str(net.ip), str(net.netmask)
+
+
+def get_net_and_prefixlen(cidr):
+    net = netaddr.IPNetwork(cidr)
+    return str(net.ip), str(net._prefixlen)
+
+
+def get_ip_version(cidr):
+    net = netaddr.IPNetwork(cidr)
+    return int(net.version)
+
+
+def get_network_info(instance):
+    # TODO(adiantum) If we will keep this function
+    # we should cache network_info
+    admin_context = context.get_admin_context()
+
+    ip_addresses = db.fixed_ip_get_all_by_instance(admin_context,
+                                                   instance['id'])
+    networks = db.network_get_all_by_instance(admin_context,
+                                              instance['id'])
+    flavor = db.instance_type_get_by_id(admin_context,
+                                        instance['instance_type_id'])
+    network_info = []
+
+    for network in networks:
+        network_ips = [ip for ip in ip_addresses
+                       if ip['network_id'] == network['id']]
+
+        def ip_dict(ip):
+            return {
+                'ip': ip['address'],
+                'netmask': network['netmask'],
+                'enabled': '1'}
+
+        def ip6_dict():
+            prefix = network['cidr_v6']
+            mac = instance['mac_address']
+            project_id = instance['project_id']
+            return  {
+                'ip': ipv6.to_global(prefix, mac, project_id),
+                'netmask': network['netmask_v6'],
+                'enabled': '1'}
+
+        mapping = {
+            'label': network['label'],
+            'gateway': network['gateway'],
+            'broadcast': network['broadcast'],
+            'mac': instance['mac_address'],
+            'rxtx_cap': flavor['rxtx_cap'],
+            'dns': [network['dns']],
+            'ips': [ip_dict(ip) for ip in network_ips]}
+
+        if FLAGS.use_ipv6:
+            mapping['ip6s'] = [ip6_dict()]
+            mapping['gateway6'] = network['gateway_v6']
+
+        network_info.append((network, mapping))
+    return network_info
diff --git a/nova/virt/vmwareapi/vmops.py b/nova/virt/vmwareapi/vmops.py
index c3e79a92f..5f76b0df5 100644
--- a/nova/virt/vmwareapi/vmops.py
+++ b/nova/virt/vmwareapi/vmops.py
@@ -150,7 +150,7 @@ class VMWareVMOps(object):
             """

             image_size, image_properties = \

                     vmware_images.get_vmdk_size_and_properties(

-                                       instance.image_id, instance)

+                                       instance.image_ref, instance)

             vmdk_file_size_in_kb = int(image_size) / 1024

             os_type = image_properties.get("vmware_ostype", "otherGuest")

             adapter_type = image_properties.get("vmware_adaptertype",

@@ -265,23 +265,23 @@ class VMWareVMOps(object):
 

         def _fetch_image_on_esx_datastore():

             """Fetch image from Glance to ESX datastore."""

-            LOG.debug(_("Downloading image file data %(image_id)s to the ESX "

+            LOG.debug(_("Downloading image file data %(image_ref)s to the ESX "

                         "data store %(data_store_name)s") %

-                        ({'image_id': instance.image_id,

+                        ({'image_ref': instance.image_ref,

                           'data_store_name': data_store_name}))

             # Upload the -flat.vmdk file whose meta-data file we just created

             # above

             vmware_images.fetch_image(

-                instance.image_id,

+                instance.image_ref,

                 instance,

                 host=self._session._host_ip,

                 data_center_name=self._get_datacenter_name_and_ref()[1],

                 datastore_name=data_store_name,

                 cookies=cookies,

                 file_path=flat_uploaded_vmdk_name)

-            LOG.debug(_("Downloaded image file data %(image_id)s to the ESX "

+            LOG.debug(_("Downloaded image file data %(image_ref)s to the ESX "

                         "data store %(data_store_name)s") %

-                        ({'image_id': instance.image_id,

+                        ({'image_ref': instance.image_ref,

                          'data_store_name': data_store_name}))

         _fetch_image_on_esx_datastore()

 

@@ -590,11 +590,11 @@ class VMWareVMOps(object):
 

     def pause(self, instance, callback):

         """Pause a VM instance."""

-        raise exception.APIError("pause not supported for vmwareapi")

+        raise exception.ApiError("pause not supported for vmwareapi")

 

     def unpause(self, instance, callback):

         """Un-Pause a VM instance."""

-        raise exception.APIError("unpause not supported for vmwareapi")

+        raise exception.ApiError("unpause not supported for vmwareapi")

 

     def suspend(self, instance, callback):

         """Suspend the specified instance."""

@@ -673,7 +673,7 @@ class VMWareVMOps(object):
 

     def get_diagnostics(self, instance):

         """Return data about VM diagnostics."""

-        raise exception.APIError("get_diagnostics not implemented for "

+        raise exception.ApiError("get_diagnostics not implemented for "

                                  "vmwareapi")

 

     def get_console_output(self, instance):

diff --git a/nova/virt/vmwareapi/vmware_images.py b/nova/virt/vmwareapi/vmware_images.py
index 50c6baedf..70adba74f 100644
--- a/nova/virt/vmwareapi/vmware_images.py
+++ b/nova/virt/vmwareapi/vmware_images.py
@@ -18,10 +18,9 @@
 Utility functions for Image transfer.

 """

 

-from glance import client

-

 from nova import exception

 from nova import flags

+import nova.image

 from nova import log as logging

 from nova.virt.vmwareapi import io_util

 from nova.virt.vmwareapi import read_write_util

@@ -91,8 +90,6 @@ def fetch_image(image, instance, **kwargs):
         func = _get_glance_image

     elif FLAGS.image_service == "nova.image.s3.S3ImageService":

         func = _get_s3_image

-    elif FLAGS.image_service == "nova.image.local.LocalImageService":

-        func = _get_local_image

     else:

         raise NotImplementedError(_("The Image Service %s is not implemented")

                                   % FLAGS.image_service)

@@ -106,8 +103,6 @@ def upload_image(image, instance, **kwargs):
         func = _put_glance_image

     elif FLAGS.image_service == "nova.image.s3.S3ImageService":

         func = _put_s3_image

-    elif FLAGS.image_service == "nova.image.local.LocalImageService":

-        func = _put_local_image

     else:

         raise NotImplementedError(_("The Image Service %s is not implemented")

                                   % FLAGS.image_service)

@@ -117,8 +112,8 @@ def upload_image(image, instance, **kwargs):
 def _get_glance_image(image, instance, **kwargs):

     """Download image from the glance image server."""

     LOG.debug(_("Downloading image %s from glance image server") % image)

-    glance_client = client.Client(FLAGS.glance_host, FLAGS.glance_port)

-    metadata, read_iter = glance_client.get_image(image)

+    (glance_client, image_id) = nova.image.get_glance_client(image)

+    metadata, read_iter = glance_client.get_image(image_id)

     read_file_handle = read_write_util.GlanceFileRead(read_iter)

     file_size = int(metadata['size'])

     write_file_handle = read_write_util.VMWareHTTPWriteFile(

@@ -153,7 +148,7 @@ def _put_glance_image(image, instance, **kwargs):
                                 kwargs.get("cookies"),

                                 kwargs.get("file_path"))

     file_size = read_file_handle.get_size()

-    glance_client = client.Client(FLAGS.glance_host, FLAGS.glance_port)

+    (glance_client, image_id) = nova.image.get_glance_client(image)

     # The properties and other fields that we need to set for the image.

     image_metadata = {"is_public": True,

                       "disk_format": "vmdk",

@@ -165,7 +160,7 @@ def _put_glance_image(image, instance, **kwargs):
                                      "vmware_image_version":

                                             kwargs.get("image_version")}}

     start_transfer(read_file_handle, file_size, glance_client=glance_client,

-                        image_id=image, image_meta=image_metadata)

+                        image_id=image_id, image_meta=image_metadata)

     LOG.debug(_("Uploaded image %s to the Glance image server") % image)

 

 

@@ -188,14 +183,11 @@ def get_vmdk_size_and_properties(image, instance):
 

     LOG.debug(_("Getting image size for the image %s") % image)

     if FLAGS.image_service == "nova.image.glance.GlanceImageService":

-        glance_client = client.Client(FLAGS.glance_host,

-                                             FLAGS.glance_port)

-        meta_data = glance_client.get_image_meta(image)

+        (glance_client, image_id) = nova.image.get_glance_client(image)

+        meta_data = glance_client.get_image_meta(image_id)

         size, properties = meta_data["size"], meta_data["properties"]

     elif FLAGS.image_service == "nova.image.s3.S3ImageService":

         raise NotImplementedError

-    elif FLAGS.image_service == "nova.image.local.LocalImageService":

-        raise NotImplementedError

     LOG.debug(_("Got image size of %(size)s for the image %(image)s") %

               locals())

     return size, properties

diff --git a/nova/virt/vmwareapi_conn.py b/nova/virt/vmwareapi_conn.py
index 1c6d2572d..3c6345ec8 100644
--- a/nova/virt/vmwareapi_conn.py
+++ b/nova/virt/vmwareapi_conn.py
@@ -124,7 +124,7 @@ class VMWareESXConnection(driver.ComputeDriver):
         """List VM instances."""

         return self._vmops.list_instances()

 

-    def spawn(self, instance):

+    def spawn(self, instance, network_info=None, block_device_mapping=None):

         """Create VM instance."""

         self._vmops.spawn(instance)

 

diff --git a/nova/virt/xenapi/fake.py b/nova/virt/xenapi/fake.py
index e36ef3288..d5ac39473 100644
--- a/nova/virt/xenapi/fake.py
+++ b/nova/virt/xenapi/fake.py
@@ -51,13 +51,13 @@ A fake XenAPI SDK.
 """
 
 
-import datetime
 import uuid
 
 from pprint import pformat
 
 from nova import exception
 from nova import log as logging
+from nova import utils
 
 
 _CLASSES = ['host', 'network', 'session', 'SR', 'VBD',
@@ -146,6 +146,7 @@ def create_vdi(name_label, read_only, sr_ref, sharable):
 def create_vbd(vm_ref, vdi_ref):
     vbd_rec = {'VM': vm_ref,
                'VDI': vdi_ref,
+               'userdevice': '0',
                'currently_attached': False}
     vbd_ref = _create_object('VBD', vbd_rec)
     after_VBD_create(vbd_ref, vbd_rec)
@@ -159,7 +160,10 @@ def after_VBD_create(vbd_ref, vbd_rec):
     vbd_rec['device'] = ''
     vm_ref = vbd_rec['VM']
     vm_rec = _db_content['VM'][vm_ref]
-    vm_rec['VBDs'] = [vbd_ref]
+    if vm_rec.get('VBDs', None):
+        vm_rec['VBDs'].append(vbd_ref)
+    else:
+        vm_rec['VBDs'] = [vbd_ref]
 
     vm_name_label = _db_content['VM'][vm_ref]['name_label']
     vbd_rec['vm_name_label'] = vm_name_label
@@ -337,10 +341,6 @@ class SessionBase(object):
             return
         db_ref['xenstore_data'][key] = None
 
-    def network_get_all_records_where(self, _1, _2):
-        # TODO (salvatore-orlando): filter table on _2
-        return _db_content['network']
-
     def VM_add_to_xenstore_data(self, _1, vm_ref, key, value):
         db_ref = _db_content['VM'][vm_ref]
         if not 'xenstore_data' in db_ref:
@@ -351,7 +351,7 @@ class SessionBase(object):
         #Always return 12GB available
         return 12 * 1024 * 1024 * 1024
 
-    def host_call_plugin(*args):
+    def host_call_plugin(self, *args):
         return 'herp'
 
     def network_get_all_records_where(self, _1, filter):
@@ -537,7 +537,7 @@ class SessionBase(object):
         except Failure, exc:
             task['error_info'] = exc.details
             task['status'] = 'failed'
-        task['finished'] = datetime.datetime.now()
+        task['finished'] = utils.utcnow()
         return task_ref
 
     def _check_session(self, params):
diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py
index 31f26e0dc..0d08e459c 100644
--- a/nova/virt/xenapi/vm_utils.py
+++ b/nova/virt/xenapi/vm_utils.py
@@ -19,6 +19,7 @@ Helper methods for operations related to the management of VM records and
 their attributes like VDIs, VIFs, as well as their lookup functions.
 """
 
+import json
 import os
 import pickle
 import re
@@ -31,6 +32,8 @@ from xml.dom import minidom
 import glance.client
 from nova import exception
 from nova import flags
+import nova.image
+from nova.image import glance as glance_image_service
 from nova import log as logging
 from nova import utils
 from nova.auth.manager import AuthManager
@@ -156,7 +159,6 @@ class VMHelper(HelperBase):
                 rec['PV_ramdisk'] = ramdisk
             else:
                 # 2. Use kernel within the image
-                rec['PV_args'] = 'clocksource=jiffies'
                 rec['PV_bootloader'] = 'pygrub'
         else:
             # 3. Using hardware virtualization
@@ -307,19 +309,16 @@ class VMHelper(HelperBase):
 
     @classmethod
     def get_vdi_for_vm_safely(cls, session, vm_ref):
-        vdi_refs = VMHelper.lookup_vm_vdis(session, vm_ref)
-        if vdi_refs is None:
-            raise Exception(_("No VDIs found for VM %s") % vm_ref)
-        else:
-            num_vdis = len(vdi_refs)
-            if num_vdis != 1:
-                raise Exception(
-                        _("Unexpected number of VDIs (%(num_vdis)s) found"
-                        " for VM %(vm_ref)s") % locals())
-
-        vdi_ref = vdi_refs[0]
-        vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref)
-        return vdi_ref, vdi_rec
+        """Retrieves the primary VDI for a VM"""
+        vbd_refs = session.get_xenapi().VM.get_VBDs(vm_ref)
+        for vbd in vbd_refs:
+            vbd_rec = session.get_xenapi().VBD.get_record(vbd)
+            # Convention dictates the primary VDI will be userdevice 0
+            if vbd_rec['userdevice'] == '0':
+                vdi_rec = session.get_xenapi().VDI.get_record(vbd_rec['VDI'])
+                return vbd_rec['VDI'], vdi_rec
+        raise exception.Error(_("No primary VDI found for"
+                "%(vm_ref)s") % locals())
 
     @classmethod
     def create_snapshot(cls, session, instance_id, vm_ref, label):
@@ -353,12 +352,6 @@ class VMHelper(HelperBase):
         return template_vm_ref, template_vdi_uuids
 
     @classmethod
-    def get_sr(cls, session, sr_label='slices'):
-        """Finds the SR named by the given name label and returns
-        the UUID"""
-        return session.call_xenapi('SR.get_by_name_label', sr_label)[0]
-
-    @classmethod
     def get_sr_path(cls, session):
         """Return the path to our storage repository
 
@@ -382,10 +375,12 @@ class VMHelper(HelperBase):
 
         os_type = instance.os_type or FLAGS.default_os_type
 
+        glance_host, glance_port = \
+            glance_image_service.pick_glance_api_server()
         params = {'vdi_uuids': vdi_uuids,
                   'image_id': image_id,
-                  'glance_host': FLAGS.glance_host,
-                  'glance_port': FLAGS.glance_port,
+                  'glance_host': glance_host,
+                  'glance_port': glance_port,
                   'sr_path': cls.get_sr_path(session),
                   'os_type': os_type}
 
@@ -419,6 +414,9 @@ class VMHelper(HelperBase):
             xenapi_image_service = ['glance', 'objectstore']
             glance_address = 'address for glance services'
             glance_port = 'port for glance services'
+
+        Returns: A single filename if image_type is KERNEL_RAMDISK
+                 A list of dictionaries that describe VDIs, otherwise
         """
         access = AuthManager().get_access_key(user, project)
 
@@ -433,6 +431,10 @@ class VMHelper(HelperBase):
     @classmethod
     def _fetch_image_glance_vhd(cls, session, instance_id, image, access,
                                 image_type):
+        """Tell glance to download an image and put the VHDs into the SR
+
+        Returns: A list of dictionaries that describe VDIs
+        """
         LOG.debug(_("Asking xapi to fetch vhd image %(image)s")
                     % locals())
 
@@ -443,26 +445,36 @@ class VMHelper(HelperBase):
         # here (under Python 2.6+) and pass them as arguments
         uuid_stack = [str(uuid.uuid4()) for i in xrange(2)]
 
+        glance_host, glance_port = \
+            glance_image_service.pick_glance_api_server()
         params = {'image_id': image,
-                  'glance_host': FLAGS.glance_host,
-                  'glance_port': FLAGS.glance_port,
+                  'glance_host': glance_host,
+                  'glance_port': glance_port,
                   'uuid_stack': uuid_stack,
                   'sr_path': cls.get_sr_path(session)}
 
         kwargs = {'params': pickle.dumps(params)}
         task = session.async_call_plugin('glance', 'download_vhd', kwargs)
-        vdi_uuid = session.wait_for_task(task, instance_id)
+        result = session.wait_for_task(task, instance_id)
+        # 'download_vhd' will return a json encoded string containing
+        # a list of dictionaries describing VDIs.  The dictionary will
+        # contain 'vdi_type' and 'vdi_uuid' keys.  'vdi_type' can be
+        # 'os' or 'swap' right now.
+        vdis = json.loads(result)
+        for vdi in vdis:
+            LOG.debug(_("xapi 'download_vhd' returned VDI of "
+                    "type '%(vdi_type)s' with UUID '%(vdi_uuid)s'" % vdi))
 
         cls.scan_sr(session, instance_id, sr_ref)
 
+        # Pull out the UUID of the first VDI
+        vdi_uuid = vdis[0]['vdi_uuid']
         # Set the name-label to ease debugging
         vdi_ref = session.get_xenapi().VDI.get_by_uuid(vdi_uuid)
-        name_label = get_name_label_for_image(image)
-        session.get_xenapi().VDI.set_name_label(vdi_ref, name_label)
+        primary_name_label = get_name_label_for_image(image)
+        session.get_xenapi().VDI.set_name_label(vdi_ref, primary_name_label)
 
-        LOG.debug(_("xapi 'download_vhd' returned VDI UUID %(vdi_uuid)s")
-                  % locals())
-        return vdi_uuid
+        return vdis
 
     @classmethod
     def _fetch_image_glance_disk(cls, session, instance_id, image, access,
@@ -474,6 +486,8 @@ class VMHelper(HelperBase):
         plugin; instead, it streams the disks through domU to the VDI
         directly.
 
+        Returns: A single filename if image_type is KERNEL_RAMDISK
+                 A list of dictionaries that describe VDIs, otherwise
         """
         # FIXME(sirp): Since the Glance plugin seems to be required for the
         # VHD disk, it may be worth using the plugin for both VHD and RAW and
@@ -485,8 +499,8 @@ class VMHelper(HelperBase):
         else:
             sr_ref = safe_find_sr(session)
 
-        client = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port)
-        meta, image_file = client.get_image(image)
+        glance_client, image_id = nova.image.get_glance_client(image)
+        meta, image_file = glance_client.get_image(image_id)
         virtual_size = int(meta['size'])
         vdi_size = virtual_size
         LOG.debug(_("Size for image %(image)s:%(virtual_size)d") % locals())
@@ -526,7 +540,8 @@ class VMHelper(HelperBase):
             LOG.debug(_("Kernel/Ramdisk VDI %s destroyed"), vdi_ref)
             return filename
         else:
-            return session.get_xenapi().VDI.get_uuid(vdi_ref)
+            vdi_uuid = session.get_xenapi().VDI.get_uuid(vdi_ref)
+            return [dict(vdi_type='os', vdi_uuid=vdi_uuid)]
 
     @classmethod
     def determine_disk_image_type(cls, instance):
@@ -547,10 +562,10 @@ class VMHelper(HelperBase):
                              ImageType.DISK_VHD: 'DISK_VHD',
                              ImageType.DISK_ISO: 'DISK_ISO'}
             disk_format = pretty_format[image_type]
-            image_id = instance.image_id
+            image_ref = instance.image_ref
             instance_id = instance.id
             LOG.debug(_("Detected %(disk_format)s format for image "
-                        "%(image_id)s, instance %(instance_id)s") % locals())
+                        "%(image_ref)s, instance %(instance_id)s") % locals())
 
         def determine_from_glance():
             glance_disk_format2nova_type = {
@@ -560,8 +575,9 @@ class VMHelper(HelperBase):
                 'raw': ImageType.DISK_RAW,
                 'vhd': ImageType.DISK_VHD,
                 'iso': ImageType.DISK_ISO}
-            client = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port)
-            meta = client.get_image_meta(instance.image_id)
+            image_ref = instance.image_ref
+            glance_client, image_id = nova.image.get_glance_client(image_ref)
+            meta = glance_client.get_image_meta(image_id)
             disk_format = meta['disk_format']
             try:
                 return glance_disk_format2nova_type[disk_format]
@@ -587,6 +603,11 @@ class VMHelper(HelperBase):
     @classmethod
     def _fetch_image_glance(cls, session, instance_id, image, access,
                             image_type):
+        """Fetch image from glance based on image type.
+
+        Returns: A single filename if image_type is KERNEL_RAMDISK
+                 A list of dictionaries that describe VDIs, otherwise
+        """
         if image_type == ImageType.DISK_VHD:
             return cls._fetch_image_glance_vhd(
                 session, instance_id, image, access, image_type)
@@ -597,7 +618,13 @@ class VMHelper(HelperBase):
     @classmethod
     def _fetch_image_objectstore(cls, session, instance_id, image, access,
                                  secret, image_type):
-        url = images.image_url(image)
+        """Fetch an image from objectstore.
+
+        Returns: A single filename if image_type is KERNEL_RAMDISK
+                 A list of dictionaries that describe VDIs, otherwise
+        """
+        url = "http://%s:%s/_images/%s/image" % (FLAGS.s3_host, FLAGS.s3_port,
+                                                 image)
         LOG.debug(_("Asking xapi to fetch %(url)s as %(access)s") % locals())
         if image_type == ImageType.KERNEL_RAMDISK:
             fn = 'get_kernel'
@@ -614,8 +641,10 @@ class VMHelper(HelperBase):
             if image_type == ImageType.DISK_RAW:
                 args['raw'] = 'true'
         task = session.async_call_plugin('objectstore', fn, args)
-        uuid = session.wait_for_task(task, instance_id)
-        return uuid
+        uuid_or_fn = session.wait_for_task(task, instance_id)
+        if image_type != ImageType.KERNEL_RAMDISK:
+            return [dict(vdi_type='os', vdi_uuid=uuid_or_fn)]
+        return uuid_or_fn
 
     @classmethod
     def determine_is_pv(cls, session, instance_id, vdi_ref, disk_image_type,
@@ -810,8 +839,7 @@ class VMHelper(HelperBase):
     @classmethod
     def scan_default_sr(cls, session):
         """Looks for the system default SR and triggers a re-scan"""
-        #FIXME(sirp/mdietz): refactor scan_default_sr in there
-        sr_ref = cls.get_sr(session)
+        sr_ref = find_sr(session)
         session.call_xenapi('SR.scan', sr_ref)
 
 
@@ -903,7 +931,8 @@ def get_vdi_for_vm_safely(session, vm_ref):
     else:
         num_vdis = len(vdi_refs)
         if num_vdis != 1:
-            raise Exception(_("Unexpected number of VDIs (%(num_vdis)s) found"
+            raise exception.Exception(_("Unexpected number of VDIs"
+                    "(%(num_vdis)s) found"
                     " for VM %(vm_ref)s") % locals())
 
     vdi_ref = vdi_refs[0]
@@ -1109,6 +1138,8 @@ def _stream_disk(dev, image_type, virtual_size, image_file):
         offset = MBR_SIZE_BYTES
         _write_partition(virtual_size, dev)
 
+    utils.execute('sudo', 'chown', os.getuid(), '/dev/%s' % dev)
+
     with open('/dev/%s' % dev, 'wb') as f:
         f.seek(offset)
         for chunk in image_file:
diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py
index 94d3b4044..5cf99b9ac 100644
--- a/nova/virt/xenapi/vmops.py
+++ b/nova/virt/xenapi/vmops.py
@@ -47,6 +47,21 @@ LOG = logging.getLogger("nova.virt.xenapi.vmops")
 FLAGS = flags.FLAGS
 
 
+def cmp_version(a, b):
+    """Compare two version strings (eg 0.0.1.10 > 0.0.1.9)"""
+    a = a.split('.')
+    b = b.split('.')
+
+    # Compare each individual portion of both version strings
+    for va, vb in zip(a, b):
+        ret = int(va) - int(vb)
+        if ret:
+            return ret
+
+    # Fallback to comparing length last
+    return len(a) - len(b)
+
+
 class VMOps(object):
     """
     Management class for VM-related tasks
@@ -91,7 +106,8 @@ class VMOps(object):
     def finish_resize(self, instance, disk_info):
         vdi_uuid = self.link_disks(instance, disk_info['base_copy'],
                 disk_info['cow'])
-        vm_ref = self._create_vm(instance, vdi_uuid)
+        vm_ref = self._create_vm(instance,
+                [dict(vdi_type='os', vdi_uuid=vdi_uuid)])
         self.resize_instance(instance, vdi_uuid)
         self._spawn(instance, vm_ref)
 
@@ -100,29 +116,30 @@ class VMOps(object):
         if not vm_ref:
             vm_ref = VMHelper.lookup(self._session, instance.name)
         if vm_ref is None:
-            raise exception(_('Attempted to power on non-existent instance'
+            raise Exception(_('Attempted to power on non-existent instance'
             ' bad instance id %s') % instance.id)
         LOG.debug(_("Starting instance %s"), instance.name)
         self._session.call_xenapi('VM.start', vm_ref, False, False)
 
-    def _create_disk(self, instance):
+    def _create_disks(self, instance):
         user = AuthManager().get_user(instance.user_id)
         project = AuthManager().get_project(instance.project_id)
         disk_image_type = VMHelper.determine_disk_image_type(instance)
-        vdi_uuid = VMHelper.fetch_image(self._session, instance.id,
-                instance.image_id, user, project, disk_image_type)
-        return vdi_uuid
+        vdis = VMHelper.fetch_image(self._session,
+                instance.id, instance.image_ref, user, project,
+                disk_image_type)
+        return vdis
 
     def spawn(self, instance, network_info=None):
-        vdi_uuid = self._create_disk(instance)
-        vm_ref = self._create_vm(instance, vdi_uuid, network_info)
+        vdis = self._create_disks(instance)
+        vm_ref = self._create_vm(instance, vdis, network_info)
         self._spawn(instance, vm_ref)
 
     def spawn_rescue(self, instance):
         """Spawn a rescue instance."""
         self.spawn(instance)
 
-    def _create_vm(self, instance, vdi_uuid, network_info=None):
+    def _create_vm(self, instance, vdis, network_info=None):
         """Create VM instance."""
         instance_name = instance.name
         vm_ref = VMHelper.lookup(self._session, instance_name)
@@ -141,43 +158,81 @@ class VMOps(object):
         user = AuthManager().get_user(instance.user_id)
         project = AuthManager().get_project(instance.project_id)
 
-        # Are we building from a pre-existing disk?
-        vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid)
-
         disk_image_type = VMHelper.determine_disk_image_type(instance)
 
         kernel = None
         if instance.kernel_id:
             kernel = VMHelper.fetch_image(self._session, instance.id,
-                instance.kernel_id, user, project, ImageType.KERNEL_RAMDISK)
+                    instance.kernel_id, user, project,
+                    ImageType.KERNEL_RAMDISK)
 
         ramdisk = None
         if instance.ramdisk_id:
             ramdisk = VMHelper.fetch_image(self._session, instance.id,
-                instance.ramdisk_id, user, project, ImageType.KERNEL_RAMDISK)
+                    instance.ramdisk_id, user, project,
+                    ImageType.KERNEL_RAMDISK)
+        # Create the VM ref and attach the first disk
+        first_vdi_ref = self._session.call_xenapi('VDI.get_by_uuid',
+                vdis[0]['vdi_uuid'])
+
+        vm_mode = instance.vm_mode and instance.vm_mode.lower()
+        if vm_mode == 'pv':
+            use_pv_kernel = True
+        elif vm_mode in ('hv', 'hvm'):
+            use_pv_kernel = False
+            vm_mode = 'hvm'  # Normalize
+        else:
+            use_pv_kernel = VMHelper.determine_is_pv(self._session,
+                    instance.id, first_vdi_ref, disk_image_type,
+                    instance.os_type)
+            vm_mode = use_pv_kernel and 'pv' or 'hvm'
+
+        if instance.vm_mode != vm_mode:
+            # Update database with normalized (or determined) value
+            db.instance_update(context.get_admin_context(),
+                               instance['id'], {'vm_mode': vm_mode})
+
+        vm_ref = VMHelper.create_vm(self._session, instance,
+                kernel, ramdisk, use_pv_kernel)
 
-        use_pv_kernel = VMHelper.determine_is_pv(self._session, instance.id,
-            vdi_ref, disk_image_type, instance.os_type)
-        vm_ref = VMHelper.create_vm(self._session, instance, kernel, ramdisk,
-                                    use_pv_kernel)
+        # device 0 reserved for RW disk
+		userdevice = 0;
 
         # DISK_ISO needs two VBDs: the ISO disk and a blank RW disk
         if disk_image_type == ImageType.DISK_ISO:
             LOG.debug("detected ISO image type, going to create blank VM for "
                   "install")
-            # device 0 reserved for RW disk, so use '1'
-            cd_vdi_ref = vdi_ref
-            VMHelper.create_cd_vbd(session=self._session, vm_ref=vm_ref,
-                    vdi_ref=cd_vdi_ref, userdevice=1, bootable=True)
 
-            vdi_ref = VMHelper.fetch_blank_disk(session=self._session,
+            cd_vdi_ref = first_vdi_ref
+            first_vdi_ref = VMHelper.fetch_blank_disk(session=self._session,
                         instance_type_id=instance.instance_type_id)
 
             VMHelper.create_vbd(session=self._session, vm_ref=vm_ref,
-                vdi_ref=vdi_ref, userdevice=0, bootable=False)
+                vdi_ref=first_vdi_ref, userdevice=userdevice, bootable=False)
+
+            # device 1 reserved for rescue disk so use '2'.
+            userdevice = userdevice + 2
+            VMHelper.create_cd_vbd(session=self._session, vm_ref=vm_ref,
+                    vdi_ref=cd_vdi_ref, userdevice=userdevice, bootable=True)
+
+
         else:
             VMHelper.create_vbd(session=self._session, vm_ref=vm_ref,
-                vdi_ref=vdi_ref, userdevice=0, bootable=True)
+                vdi_ref=first_vdi_ref, userdevice=userdevice, bootable=True)
+            # userdevice 1 is reserved for rescue
+            userdevice = userdevice + 1
+
+
+        # Attach any other disks
+        for vdi in vdis[1:]:
+            # vdi['vdi_type'] is either 'os' or 'swap', but we don't
+            # really care what it is right here.
+            vdi_ref = self._session.call_xenapi('VDI.get_by_uuid',
+                    vdi['vdi_uuid'])
+            VMHelper.create_vbd(session=self._session, vm_ref=vm_ref,
+                    vdi_ref=vdi_ref, userdevice=userdevice,
+                    bootable=False)
+            userdevice += 1
 
         # TODO(tr3buchet) - check to make sure we have network info, otherwise
         # create it now. This goes away once nova-multi-nic hits.
@@ -187,7 +242,7 @@ class VMOps(object):
         # Alter the image before VM start for, e.g. network injection
         if FLAGS.xenapi_inject_image:
             VMHelper.preconfigure_instance(self._session, instance,
-                                           vdi_ref, network_info)
+                                           first_vdi_ref, network_info)
 
         self.create_vifs(vm_ref, network_info)
         self.inject_network_info(instance, network_info, vm_ref)
@@ -201,6 +256,34 @@ class VMOps(object):
         LOG.info(_('Spawning VM %(instance_name)s created %(vm_ref)s.')
                  % locals())
 
+        ctx = context.get_admin_context()
+        agent_build = db.agent_build_get_by_triple(ctx, 'xen',
+                              instance.os_type, instance.architecture)
+        if agent_build:
+            LOG.info(_('Latest agent build for %(hypervisor)s/%(os)s' + \
+                       '/%(architecture)s is %(version)s') % agent_build)
+        else:
+            LOG.info(_('No agent build found for %(hypervisor)s/%(os)s' + \
+                       '/%(architecture)s') % {
+                        'hypervisor': 'xen',
+                        'os': instance.os_type,
+                        'architecture': instance.architecture})
+
+        def _check_agent_version():
+            version = self.get_agent_version(instance)
+            if not version:
+                LOG.info(_('No agent version returned by instance'))
+                return
+
+            LOG.info(_('Instance agent version: %s') % version)
+            if not agent_build:
+                return
+
+            if cmp_version(version, agent_build['version']) < 0:
+                LOG.info(_('Updating Agent to %s') % agent_build['version'])
+                self.agent_update(instance, agent_build['url'],
+                              agent_build['md5hash'])
+
         def _inject_files():
             injected_files = instance.injected_files
             if injected_files:
@@ -217,6 +300,13 @@ class VMOps(object):
                 for path, contents in instance.injected_files:
                     LOG.debug(_("Injecting file path: '%s'") % path)
                     self.inject_file(instance, path, contents)
+
+        def _set_admin_password():
+            admin_password = instance.admin_pass
+            if admin_password:
+                LOG.debug(_("Setting admin password"))
+                self.set_admin_password(instance, admin_password)
+
         # NOTE(armando): Do we really need to do this in virt?
         # NOTE(tr3buchet): not sure but wherever we do it, we need to call
         #                  reset_network afterwards
@@ -228,7 +318,9 @@ class VMOps(object):
                 if state == power_state.RUNNING:
                     LOG.debug(_('Instance %s: booted'), instance_name)
                     timer.stop()
+                    _check_agent_version()
                     _inject_files()
+                    _set_admin_password()
                     return True
             except Exception, exc:
                 LOG.warn(exc)
@@ -268,7 +360,8 @@ class VMOps(object):
             instance_name = instance_or_vm.name
         vm_ref = VMHelper.lookup(self._session, instance_name)
         if vm_ref is None:
-            raise exception.InstanceNotFound(instance_id=instance_obj.id)
+            raise exception.NotFound(_("No opaque_ref could be determined "
+                    "for '%s'.") % instance_or_vm)
         return vm_ref
 
     def _acquire_bootlock(self, vm):
@@ -432,6 +525,34 @@ class VMOps(object):
         task = self._session.call_xenapi('Async.VM.clean_reboot', vm_ref)
         self._session.wait_for_task(task, instance.id)
 
+    def get_agent_version(self, instance):
+        """Get the version of the agent running on the VM instance."""
+
+        # Send the encrypted password
+        transaction_id = str(uuid.uuid4())
+        args = {'id': transaction_id}
+        resp = self._make_agent_call('version', instance, '', args)
+        if resp is None:
+            # No response from the agent
+            return
+        resp_dict = json.loads(resp)
+        return resp_dict['message']
+
+    def agent_update(self, instance, url, md5sum):
+        """Update agent on the VM instance."""
+
+        # Send the encrypted password
+        transaction_id = str(uuid.uuid4())
+        args = {'id': transaction_id, 'url': url, 'md5sum': md5sum}
+        resp = self._make_agent_call('agentupdate', instance, '', args)
+        if resp is None:
+            # No response from the agent
+            return
+        resp_dict = json.loads(resp)
+        if resp_dict['returncode'] != '0':
+            raise RuntimeError(resp_dict['message'])
+        return resp_dict['message']
+
     def set_admin_password(self, instance, new_pass):
         """Set the root/admin password on the VM instance.
 
@@ -472,6 +593,9 @@ class VMOps(object):
         # Successful return code from password is '0'
         if resp_dict['returncode'] != '0':
             raise RuntimeError(resp_dict['message'])
+        db.instance_update(context.get_admin_context(),
+                                  instance['id'],
+                                  dict(admin_pass=new_pass))
         return resp_dict['message']
 
     def inject_file(self, instance, path, contents):
@@ -1176,16 +1300,12 @@ class SimpleDH(object):
         mpi = M2Crypto.m2.bn_to_mpi(bn)
         return mpi
 
-    def _run_ssl(self, text, which):
-        base_cmd = ('openssl enc -aes-128-cbc -a -pass pass:%(shared)s '
-                '-nosalt %(dec_flag)s')
-        if which.lower()[0] == 'd':
-            dec_flag = ' -d'
-        else:
-            dec_flag = ''
-        shared = self._shared
-        cmd = base_cmd % locals()
-        proc = _runproc(cmd)
+    def _run_ssl(self, text, extra_args=None):
+        if not extra_args:
+            extra_args = ''
+        cmd = 'enc -aes-128-cbc -A -a -pass pass:%s -nosalt %s' % (
+                self._shared, extra_args)
+        proc = _runproc('openssl %s' % cmd)
         proc.stdin.write(text)
         proc.stdin.close()
         proc.wait()
@@ -1195,7 +1315,7 @@ class SimpleDH(object):
         return proc.stdout.read()
 
     def encrypt(self, text):
-        return self._run_ssl(text, 'enc')
+        return self._run_ssl(text).strip('\n')
 
     def decrypt(self, text):
-        return self._run_ssl(text, 'dec')
+        return self._run_ssl(text, '-d')
diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py
index 6d828e109..5fcec1715 100644
--- a/nova/virt/xenapi_conn.py
+++ b/nova/virt/xenapi_conn.py
@@ -194,7 +194,7 @@ class XenAPIConnection(driver.ComputeDriver):
     def list_instances_detail(self):
         return self._vmops.list_instances_detail()
 
-    def spawn(self, instance):
+    def spawn(self, instance, network_info=None, block_device_mapping=None):
         """Create VM instance"""
         self._vmops.spawn(instance)