trunk merge, getting fierce..

17 files changed, 134 insertions, 42 deletions

diff --git a/bin/nova-manage b/bin/nova-manage
index 9d67ed95b..0d560ec07 100755
--- a/bin/nova-manage
+++ b/bin/nova-manage
@@ -56,11 +56,11 @@
 import gettext
 import glob
 import json
+import netaddr
 import os
 import sys
 import time
 
-import IPy
 
 # If ../nova/__init__.py exists, add ../ to Python search path, so that
 # it will override what happens to be installed in /usr/(local/)lib/python...
@@ -525,14 +525,14 @@ class FloatingIpCommands(object):
     def create(self, range):
         """Creates floating ips for zone by range
         arguments: ip_range"""
-        for address in IPy.IP(range):
+        for address in netaddr.IPNetwork(range):
             db.floating_ip_create(context.get_admin_context(),
                                   {'address': str(address)})
 
     def delete(self, ip_range):
         """Deletes floating ips by range
         arguments: range"""
-        for address in IPy.IP(ip_range):
+        for address in netaddr.IPNetwork(ip_range):
             db.floating_ip_destroy(context.get_admin_context(),
                                    str(address))
 
diff --git a/nova/api/ec2/admin.py b/nova/api/ec2/admin.py
index 343bc61c4..df7876b9d 100644
--- a/nova/api/ec2/admin.py
+++ b/nova/api/ec2/admin.py
@@ -369,3 +369,23 @@ class AdminController(object):
             raise exception.ApiError(_('Duplicate rule'))
         self.compute_api.trigger_provider_fw_rules_refresh(context)
         return {'status': 'OK', 'message': 'Added %s rules' % rules_added}
+
+    def describe_external_address_blocks(self, context):
+        blocks = db.provider_fw_rule_get_all(context)
+        # NOTE(todd): use a set since we have icmp/udp/tcp rules with same cidr
+        blocks = set([b.cidr for b in blocks])
+        blocks = [{'cidr': b} for b in blocks]
+        return {'externalIpBlockInfo':
+                list(sorted(blocks, key=lambda k: k['cidr']))}
+
+    def remove_external_address_block(self, context, cidr):
+        LOG.audit(_('Removing ip block from %s'), cidr, context=context)
+        cidr = urllib.unquote(cidr).decode()
+        # raise if invalid
+        netaddr.IPNetwork(cidr)
+        rules = db.provider_fw_rule_get_all_by_cidr(context, cidr)
+        for rule in rules:
+            db.provider_fw_rule_destroy(context, rule['id'])
+        if rules:
+            self.compute_api.trigger_provider_fw_rules_refresh(context)
+        return {'status': 'OK', 'message': 'Deleted %s rules' % len(rules)}
diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py
index 078d35887..e7f94830f 100644
--- a/nova/api/ec2/cloud.py
+++ b/nova/api/ec2/cloud.py
@@ -23,7 +23,7 @@ datastore.
 """
 
 import base64
-import IPy
+import netaddr
 import os
 import urllib
 import tempfile
@@ -452,7 +452,7 @@ class CloudController(object):
         elif cidr_ip:
             # If this fails, it throws an exception. This is what we want.
             cidr_ip = urllib.unquote(cidr_ip).decode()
-            IPy.IP(cidr_ip)
+            netaddr.IPNetwork(cidr_ip)
             values['cidr'] = cidr_ip
         else:
             values['cidr'] = '0.0.0.0/0'
diff --git a/nova/db/api.py b/nova/db/api.py
index a3a3d2b6b..b372ac6a7 100644
--- a/nova/db/api.py
+++ b/nova/db/api.py
@@ -1094,6 +1094,16 @@ def provider_fw_rule_get_all(context):
     return IMPL.provider_fw_rule_get_all(context)
 
 
+def provider_fw_rule_get_all_by_cidr(context, cidr):
+    """Get all provider-level firewall rules."""
+    return IMPL.provider_fw_rule_get_all_by_cidr(context, cidr)
+
+
+def provider_fw_rule_destroy(context, rule_id):
+    """Delete a provider firewall rule from the database."""
+    return IMPL.provider_fw_rule_destroy(context, rule_id)
+
+
 ###################
 
 
diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py
index c3de412d7..d13efb874 100644
--- a/nova/db/sqlalchemy/api.py
+++ b/nova/db/sqlalchemy/api.py
@@ -2421,6 +2421,7 @@ def provider_fw_rule_create(context, rule):
     return fw_rule_ref
 
 
+@require_admin_context
 def provider_fw_rule_get_all(context):
     session = get_session()
     return session.query(models.ProviderFirewallRule).\
@@ -2428,6 +2429,26 @@ def provider_fw_rule_get_all(context):
                    all()
 
 
+@require_admin_context
+def provider_fw_rule_get_all_by_cidr(context, cidr):
+    session = get_session()
+    return session.query(models.ProviderFirewallRule).\
+                   filter_by(deleted=can_read_deleted(context)).\
+                   filter_by(cidr=cidr).\
+                   all()
+
+
+@require_admin_context
+def provider_fw_rule_destroy(context, rule_id):
+    session = get_session()
+    with session.begin():
+        session.query(models.ProviderFirewallRule).\
+                filter_by(id=rule_id).\
+                update({'deleted': True,
+                        'deleted_at': utils.utcnow(),
+                        'updated_at': literal_column('updated_at')})
+
+
 ###################
 
 
diff --git a/nova/image/glance.py b/nova/image/glance.py
index 6e058ab2f..55d948a32 100644
--- a/nova/image/glance.py
+++ b/nova/image/glance.py
@@ -59,7 +59,7 @@ class GlanceImageService(service.BaseImageService):
     """Provides storage and retrieval of disk image objects within Glance."""
 
     GLANCE_ONLY_ATTRS = ['size', 'location', 'disk_format',
-                         'container_format']
+                         'container_format', 'checksum']
 
     # NOTE(sirp): Overriding to use _translate_to_service provided by
     # BaseImageService
diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py
index 9a5f2c5e6..283a5aca1 100644
--- a/nova/network/linux_net.py
+++ b/nova/network/linux_net.py
@@ -20,6 +20,7 @@
 
 import calendar
 import inspect
+import netaddr
 import os
 
 from nova import db
@@ -27,7 +28,6 @@ from nova import exception
 from nova import flags
 from nova import log as logging
 from nova import utils
-from IPy import IP
 
 
 LOG = logging.getLogger("nova.linux_net")
@@ -707,7 +707,7 @@ def _dnsmasq_cmd(net):
            '--listen-address=%s' % net['gateway'],
            '--except-interface=lo',
            '--dhcp-range=%s,static,120s' % net['dhcp_start'],
-           '--dhcp-lease-max=%s' % IP(net['cidr']).len(),
+           '--dhcp-lease-max=%s' % len(netaddr.IPNetwork(net['cidr'])),
            '--dhcp-hostsfile=%s' % _dhcp_file(net['bridge'], 'conf'),
            '--dhcp-script=%s' % FLAGS.dhcpbridge,
            '--leasefile-ro']
diff --git a/nova/network/manager.py b/nova/network/manager.py
index 100b5ddc0..b60e70990 100644
--- a/nova/network/manager.py
+++ b/nova/network/manager.py
@@ -47,12 +47,11 @@ topologies.  All of the network commands are issued to a subclass of
 
 import datetime
 import math
+import netaddr
 import socket
 import pickle
 from eventlet import greenpool
 
-import IPy
-
 from nova import context
 from nova import db
 from nova import exception
@@ -550,8 +549,8 @@ class NetworkManager(manager.SchedulerDependentManager):
                         network_size, cidr_v6, gateway_v6, bridge,
                         bridge_interface, **kwargs):
         """Create networks based on parameters."""
-        fixed_net = IPy.IP(cidr)
-        fixed_net_v6 = IPy.IP(cidr_v6)
+        fixed_net = netaddr.IPNetwork(cidr)
+        fixed_net_v6 = netaddr.IPNetwork(cidr_v6)
         significant_bits_v6 = 64
         network_size_v6 = 1 << 64
         for index in range(num_networks):
@@ -559,16 +558,16 @@ class NetworkManager(manager.SchedulerDependentManager):
             start_v6 = index * network_size_v6
             significant_bits = 32 - int(math.log(network_size, 2))
             cidr = '%s/%s' % (fixed_net[start], significant_bits)
-            project_net = IPy.IP(cidr)
+            project_net = netaddr.IPNetwork(cidr)
             net = {}
             net['bridge'] = bridge
             net['bridge_interface'] = bridge_interface
             net['dns'] = FLAGS.flat_network_dns
             net['cidr'] = cidr
-            net['netmask'] = str(project_net.netmask())
-            net['gateway'] = str(project_net[1])
-            net['broadcast'] = str(project_net.broadcast())
-            net['dhcp_start'] = str(project_net[2])
+            net['netmask'] = str(project_net.netmask)
+            net['gateway'] = str(list(project_net)[1])
+            net['broadcast'] = str(project_net.broadcast)
+            net['dhcp_start'] = str(list(project_net)[2])
             if num_networks > 1:
                 net['label'] = '%s_%d' % (label, index)
             else:
@@ -578,15 +577,16 @@ class NetworkManager(manager.SchedulerDependentManager):
                 cidr_v6 = '%s/%s' % (fixed_net_v6[start_v6],
                                      significant_bits_v6)
                 net['cidr_v6'] = cidr_v6
-                project_net_v6 = IPy.IP(cidr_v6)
+
+                project_net_v6 = netaddr.IPNetwork(cidr_v6)
 
                 if gateway_v6:
                     # use a pre-defined gateway if one is provided
-                    net['gateway_v6'] = str(gateway_v6)
+                    net['gateway_v6'] = str(list(gateway_v6)[1])
                 else:
-                    net['gateway_v6'] = str(project_net_v6[1])
+                    net['gateway_v6'] = str(list(project_net_v6)[1])
 
-                net['netmask_v6'] = str(project_net_v6.prefixlen())
+                net['netmask_v6'] = str(project_net_v6._prefixlen)
 
             if kwargs.get('vpn', False):
                 # this bit here is for vlan-manager
@@ -627,7 +627,7 @@ class NetworkManager(manager.SchedulerDependentManager):
         #             to properties of the manager class?
         bottom_reserved = self._bottom_reserved_ips
         top_reserved = self._top_reserved_ips
-        project_net = IPy.IP(network['cidr'])
+        project_net = netaddr.IPNetwork(network['cidr'])
         num_ips = len(project_net)
         for index in range(num_ips):
             address = str(project_net[index])
@@ -845,8 +845,8 @@ class VlanManager(RPCAllocateFixedIP, FloatingIP, NetworkManager):
                                ' than 4094'))
 
         # check that num networks and network size fits in fixed_net
-        fixed_net = IPy.IP(kwargs['cidr'])
-        if fixed_net.len() < kwargs['num_networks'] * kwargs['network_size']:
+        fixed_net = netaddr.IPNetwork(kwargs['cidr'])
+        if len(fixed_net) < kwargs['num_networks'] * kwargs['network_size']:
             raise ValueError(_('The network range is not big enough to fit '
                   '%(num_networks)s. Network size is %(network_size)s') %
                   kwargs)
diff --git a/nova/network/xenapi_net.py b/nova/network/xenapi_net.py
index 34a598ead..e86f4017d 100644
--- a/nova/network/xenapi_net.py
+++ b/nova/network/xenapi_net.py
@@ -56,8 +56,10 @@ def ensure_vlan_bridge(vlan_num, bridge, bridge_interface, net_attrs=None):
                        'other_config': {}}
         network_ref = session.call_xenapi('network.create', network_rec)
         # 2 - find PIF for VLAN
-        expr = "field 'device' = '%s' and \
-                field 'VLAN' = '-1'" % bridge_interface
+        # NOTE(salvatore-orlando): using double quotes inside single quotes
+        # as xapi filter only support tokens in double quotes
+        expr = 'field "device" = "%s" and \
+                field "VLAN" = "-1"' % bridge_interface
         pifs = session.call_xenapi('PIF.get_all_records_where', expr)
         pif_ref = None
         # Multiple PIF are ok: we are dealing with a pool
diff --git a/nova/tests/api/openstack/test_image_metadata.py b/nova/tests/api/openstack/test_image_metadata.py
index efed44960..730af3665 100644
--- a/nova/tests/api/openstack/test_image_metadata.py
+++ b/nova/tests/api/openstack/test_image_metadata.py
@@ -38,6 +38,7 @@ class ImageMetaDataTest(unittest.TestCase):
         'name': 'image1',
         'deleted': False,
         'container_format': None,
+        'checksum': None,
         'created_at': '2011-03-22T17:40:15',
         'disk_format': None,
         'updated_at': '2011-03-22T17:40:15',
@@ -53,6 +54,7 @@ class ImageMetaDataTest(unittest.TestCase):
         'name': 'image2',
         'deleted': False,
         'container_format': None,
+        'checksum': None,
         'created_at': '2011-03-22T17:40:15',
         'disk_format': None,
         'updated_at': '2011-03-22T17:40:15',
@@ -68,6 +70,7 @@ class ImageMetaDataTest(unittest.TestCase):
         'name': 'image3',
         'deleted': False,
         'container_format': None,
+        'checksum': None,
         'created_at': '2011-03-22T17:40:15',
         'disk_format': None,
         'updated_at': '2011-03-22T17:40:15',
@@ -104,7 +107,10 @@ class ImageMetaDataTest(unittest.TestCase):
         res = req.get_response(fakes.wsgi_app())
         res_dict = json.loads(res.body)
         self.assertEqual(200, res.status_int)
-        self.assertEqual('value1', res_dict['metadata']['key1'])
+        expected = self.IMAGE_FIXTURES[0]['properties']
+        self.assertEqual(len(expected), len(res_dict['metadata']))
+        for (key, value) in res_dict['metadata'].items():
+            self.assertEqual(value, res_dict['metadata'][key])
 
     def test_index_xml(self):
         serializer = openstack.image_metadata.ImageMetadataXMLSerializer()
diff --git a/nova/tests/test_adminapi.py b/nova/tests/test_adminapi.py
index eb4466781..877cf4ea1 100644
--- a/nova/tests/test_adminapi.py
+++ b/nova/tests/test_adminapi.py
@@ -81,5 +81,27 @@ class AdminApiTestCase(test.TestCase):
     def test_block_external_ips(self):
         """Make sure provider firewall rules are created."""
         result = self.api.block_external_addresses(self.context, '1.1.1.1/32')
+        self.api.remove_external_address_block(self.context, '1.1.1.1/32')
         self.assertEqual('OK', result['status'])
         self.assertEqual('Added 3 rules', result['message'])
+
+    def test_list_blocked_ips(self):
+        """Make sure we can see the external blocks that exist."""
+        self.api.block_external_addresses(self.context, '1.1.1.2/32')
+        result = self.api.describe_external_address_blocks(self.context)
+        num = len(db.provider_fw_rule_get_all(self.context))
+        self.api.remove_external_address_block(self.context, '1.1.1.2/32')
+        # we only list IP, not tcp/udp/icmp rules
+        self.assertEqual(num / 3, len(result['externalIpBlockInfo']))
+
+    def test_remove_ip_block(self):
+        """Remove ip blocks."""
+        result = self.api.block_external_addresses(self.context, '1.1.1.3/32')
+        self.assertEqual('OK', result['status'])
+        num0 = len(db.provider_fw_rule_get_all(self.context))
+        result = self.api.remove_external_address_block(self.context,
+                                                        '1.1.1.3/32')
+        self.assertEqual('OK', result['status'])
+        self.assertEqual('Deleted 3 rules', result['message'])
+        num1 = len(db.provider_fw_rule_get_all(self.context))
+        self.assert_(num1 < num0)
diff --git a/nova/tests/test_libvirt.py b/nova/tests/test_libvirt.py
index 176c536b9..5a081ddf7 100644
--- a/nova/tests/test_libvirt.py
+++ b/nova/tests/test_libvirt.py
@@ -1145,6 +1145,13 @@ class IptablesFirewallTestCase(test.TestCase):
                 provjump_rules.append(rule)
         self.assertEqual(1, len(provjump_rules))
 
+        # remove a rule from the db, cast to compute to refresh rule
+        db.provider_fw_rule_destroy(admin_ctxt, provider_fw1['id'])
+        self.fw.refresh_provider_fw_rules()
+        rules = [rule for rule in self.fw.iptables.ipv4['filter'].rules
+                      if rule.chain == 'provider']
+        self.assertEqual(1, len(rules))
+
 
 class NWFilterTestCase(test.TestCase):
     def setUp(self):
diff --git a/nova/utils.py b/nova/utils.py
index b49849f9a..4a66dd993 100644
--- a/nova/utils.py
+++ b/nova/utils.py
@@ -518,6 +518,16 @@ def loads(s):
     return json.loads(s)
 
 
+try:
+    import anyjson
+except ImportError:
+    pass
+else:
+    anyjson._modules.append(("nova.utils", "dumps", TypeError,
+                                           "loads", ValueError))
+    anyjson.force_implementation("nova.utils")
+
+
 _semaphores = {}
 
 
diff --git a/nova/virt/libvirt/connection.py b/nova/virt/libvirt/connection.py
index 394d60f71..785a83584 100644
--- a/nova/virt/libvirt/connection.py
+++ b/nova/virt/libvirt/connection.py
@@ -38,6 +38,7 @@ Supports KVM, LXC, QEMU, UML, and XEN.
 
 import hashlib
 import multiprocessing
+import netaddr
 import os
 import random
 import re
@@ -53,8 +54,6 @@ from xml.etree import ElementTree
 from eventlet import greenthread
 from eventlet import tpool
 
-import IPy
-
 from nova import context
 from nova import db
 from nova import exception
diff --git a/nova/virt/libvirt/netutils.py b/nova/virt/libvirt/netutils.py
index 773f0a09c..e5aaf7cec 100644
--- a/nova/virt/libvirt/netutils.py
+++ b/nova/virt/libvirt/netutils.py
@@ -21,7 +21,7 @@
 """Network-releated utilities for supporting libvirt connection code."""
 
 
-import IPy
+import netaddr
 
 from nova import context
 from nova import db
@@ -34,18 +34,18 @@ FLAGS = flags.FLAGS
 
 
 def get_net_and_mask(cidr):
-    net = IPy.IP(cidr)
-    return str(net.net()), str(net.netmask())
+    net = netaddr.IPNetwork(cidr)
+    return str(net.ip), str(net.netmask)
 
 
 def get_net_and_prefixlen(cidr):
-    net = IPy.IP(cidr)
-    return str(net.net()), str(net.prefixlen())
+    net = netaddr.IPNetwork(cidr)
+    return str(net.ip), str(net._prefixlen)
 
 
 def get_ip_version(cidr):
-    net = IPy.IP(cidr)
-    return int(net.version())
+    net = netaddr.IPNetwork(cidr)
+    return int(net.version)
 
 
 def get_network_info(instance):
diff --git a/plugins/xenserver/networking/etc/xensource/scripts/vif_5.6-fp1.patch b/plugins/xenserver/networking/etc/xensource/scripts/vif_5.6-fp1.patch
index feaf1312d..d42a11eff 100644
--- a/plugins/xenserver/networking/etc/xensource/scripts/vif_5.6-fp1.patch
+++ b/plugins/xenserver/networking/etc/xensource/scripts/vif_5.6-fp1.patch
@@ -8,7 +8,7 @@
      fi
      ;;
  
-@@ -224,9 +225,11 @@
+@@ -224,6 +225,7 @@
  
  remove)
      if [ "${TYPE}" = "vif" ] ;then
@@ -16,7 +16,3 @@
          xenstore-rm "${HOTPLUG}/hotplug"
      fi
      logger -t scripts-vif "${dev} has been removed"
-     remove_from_bridge
-     ;;
- esac
-+
diff --git a/tools/pip-requires b/tools/pip-requires
index 0da473109..5d31a814d 100644
--- a/tools/pip-requires
+++ b/tools/pip-requires
@@ -1,7 +1,6 @@
 SQLAlchemy==0.6.3
 pep8==0.5.0
 pylint==0.19
-IPy==0.70
 Cheetah==2.4.4
 M2Crypto==0.20.2
 amqplib==0.6.1