Implement BP untie-nova-network-models

Fixes LP853979
Remove the FK references for network data.
Remove unused db functions that used the FK's
Update db functions to not joinload
Update notification to optionally take network_info if compute has it
Update EC2 Api to use the network cache, falling back to rpc.call
Remove test_instance_get_project_vpn_joins which tests calls not used

Change-Id: I1a01ccc5ebcf7efeafe014af62be893325bb0825

5 files changed, 168 insertions, 77 deletions

diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py
index 8049d3bd7..bdeb7dc2e 100644
--- a/nova/api/ec2/cloud.py
+++ b/nova/api/ec2/cloud.py
@@ -1217,20 +1217,17 @@ class CloudController(object):
 
     def format_addresses(self, context):
         addresses = []
-        if context.is_admin:
-            iterator = db.floating_ip_get_all(context)
-        else:
-            iterator = db.floating_ip_get_all_by_project(context,
-                                                         context.project_id)
-        for floating_ip_ref in iterator:
+        floaters = self.network_api.get_floating_ips_by_project(context)
+        for floating_ip_ref in floaters:
             if floating_ip_ref['project_id'] is None:
                 continue
             address = floating_ip_ref['address']
             ec2_id = None
-            if (floating_ip_ref['fixed_ip']
-                and floating_ip_ref['fixed_ip']['instance']):
-                instance_id = floating_ip_ref['fixed_ip']['instance']['id']
-                ec2_id = ec2utils.id_to_ec2_id(instance_id)
+            if floating_ip_ref['fixed_ip_id']:
+                fixed_id = floating_ip_ref['fixed_ip_id']
+                fixed = self.network_api.get_fixed_ip(context, fixed_id)
+                if fixed['instance_id'] is not None:
+                    ec2_id = ec2utils.id_to_ec2_id(fixed['instance_id'])
             address_rv = {'public_ip': address,
                           'instance_id': ec2_id}
             if context.is_admin:
diff --git a/nova/api/ec2/ec2utils.py b/nova/api/ec2/ec2utils.py
index f54a9f960..8f641e189 100644
--- a/nova/api/ec2/ec2utils.py
+++ b/nova/api/ec2/ec2utils.py
@@ -20,8 +20,9 @@ import re
 
 from nova import exception
 from nova import flags
-from nova import ipv6
 from nova import log as logging
+from nova import network
+from nova.network import model as network_model
 
 
 FLAGS = flags.FLAGS
@@ -64,36 +65,69 @@ def image_ec2_id(image_id, image_type='ami'):
         return "ami-00000000"
 
 
-def get_ip_info_for_instance(context, instance):
-    """Return a list of all fixed IPs for an instance"""
+def get_ip_info_for_instance_from_cache(instance):
+    if (not instance.get('info_cache') or
+        not instance['info_cache'].get('network_info')):
+        # NOTE(jkoelker) Raising ValueError so that we trigger the
+        #                fallback lookup
+        raise ValueError
 
+    cached_info = instance['info_cache']['network_info']
+    nw_info = network_model.NetworkInfo.hydrate(cached_info)
     ip_info = dict(fixed_ips=[], fixed_ip6s=[], floating_ips=[])
 
-    fixed_ips = instance['fixed_ips']
-    for fixed_ip in fixed_ips:
-        fixed_addr = fixed_ip['address']
-        network = fixed_ip.get('network')
-        vif = fixed_ip.get('virtual_interface')
-        if not network or not vif:
-            name = instance['name']
-            ip = fixed_ip['address']
-            LOG.warn(_("Instance %(name)s has stale IP "
-                    "address: %(ip)s (no network or vif)") % locals())
-            continue
-        cidr_v6 = network.get('cidr_v6')
-        if FLAGS.use_ipv6 and cidr_v6:
-            ipv6_addr = ipv6.to_global(cidr_v6, vif['address'],
-                    network['project_id'])
-            if ipv6_addr not in ip_info['fixed_ip6s']:
-                ip_info['fixed_ip6s'].append(ipv6_addr)
-
-        for floating_ip in fixed_ip.get('floating_ips', []):
-            float_addr = floating_ip['address']
-            ip_info['floating_ips'].append(float_addr)
-        ip_info['fixed_ips'].append(fixed_addr)
+    for vif in nw_info:
+        vif_fixed_ips = vif.fixed_ips()
+
+        fixed_ips = [ip['address']
+                     for ip in vif_fixed_ips if ip['version'] == 4]
+        fixed_ip6s = [ip['address']
+                      for ip in vif_fixed_ips if ip['version'] == 6]
+        floating_ips = [ip['address']
+                        for ip in vif.floating_ips()]
+        ip_info['fixed_ips'].extend(fixed_ips)
+        ip_info['fixed_ip6s'].extend(fixed_ip6s)
+        ip_info['floating_ips'].extend(floating_ips)
+
+    return ip_info
+
+
+def get_ip_for_instance_from_nwinfo(context, instance):
+    # NOTE(jkoelker) When the network_api starts returning the model, this
+    #                can be refactored out into the above function
+    network_api = network.API()
+
+    def _get_floaters(ip):
+        return network_api.get_floating_ips_by_fixed_address(context, ip)
+
+    ip_info = dict(fixed_ips=[], fixed_ip6s=[], floating_ips=[])
+    nw_info = network_api.get_instance_nw_info(context, instance)
+
+    for _net, info in nw_info:
+        for ip in info['ips']:
+            ip_info['fixed_ips'].append(ip['ip'])
+            floaters = _get_floaters(ip['ip'])
+            if floaters:
+                ip_info['floating_ips'].extend(floaters)
+        if 'ip6s' in info:
+            for ip in info['ip6s']:
+                ip_info['fixed_ip6s'].append(ip['ip'])
     return ip_info
 
 
+def get_ip_info_for_instance(context, instance):
+    """Return a list of all fixed IPs for an instance"""
+
+    try:
+        return get_ip_info_for_instance_from_cache(instance)
+    except (ValueError, KeyError, AttributeError):
+        # NOTE(jkoelker) If the json load (ValueError) or the
+        #                sqlalchemy FK (KeyError, AttributeError)
+        #                fail fall back to calling out to he
+        #                network api
+        return get_ip_for_instance_from_nwinfo(context, instance)
+
+
 def get_availability_zone_by_host(services, host):
     if len(services) > 0:
         return services[0]['availability_zone']
diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py
index 5df2844ac..0ac46d96e 100644
--- a/nova/api/openstack/common.py
+++ b/nova/api/openstack/common.py
@@ -28,8 +28,9 @@ from nova.api.openstack import xmlutil
 from nova.compute import vm_states
 from nova.compute import task_states
 from nova import flags
-from nova import ipv6
 from nova import log as logging
+from nova import network
+from nova.network import model as network_model
 from nova import quota
 
 
@@ -286,6 +287,63 @@ def dict_to_query_str(params):
     return param_str.rstrip('&')
 
 
+def get_networks_for_instance_from_cache(instance):
+    if (not instance.get('info_cache') or
+        not instance['info_cache'].get('network_info')):
+        # NOTE(jkoelker) Raising ValueError so that we trigger the
+        #                fallback lookup
+        raise ValueError
+
+    cached_info = instance['info_cache']['network_info']
+    nw_info = network_model.NetworkInfo.hydrate(cached_info)
+    networks = {}
+
+    for vif in nw_info:
+        ips = vif.fixed_ips()
+        floaters = vif.floating_ips()
+        label = vif['network']['label']
+        if label not in networks:
+            networks[label] = {'ips': [], 'floating_ips': []}
+
+        networks[label]['ips'].extend(ips)
+        networks[label]['floating_ips'].extend(floaters)
+    return networks
+
+
+def get_networks_for_instance_from_nwinfo(context, instance):
+    # NOTE(jkoelker) When the network_api starts returning the model, this
+    #                can be refactored out into the above function
+    network_api = network.API()
+
+    def _get_floats(ip):
+        return network_api.get_floating_ips_by_fixed_address(context, ip)
+
+    def _emit_addr(ip, version):
+        return {'address': ip, 'version': version}
+
+    nw_info = network_api.get_instance_nw_info(context, instance)
+    networks = {}
+    for _net, info in nw_info:
+        net = {'ips': [], 'floating_ips': []}
+        for ip in info['ips']:
+            net['ips'].append(_emit_addr(ip['ip'], 4))
+            floaters = _get_floats(ip['ip'])
+            if floaters:
+                net['floating_ips'].extend([_emit_addr(float, 4)
+                                            for float in floaters])
+        if 'ip6s' in info:
+            for ip in info['ip6s']:
+                net['ips'].append(_emit_addr(ip['ip'], 6))
+
+        label = info['label']
+        if label not in networks:
+            networks[label] = {'ips': [], 'floating_ips': []}
+
+        networks[label]['ips'].extend(net['ips'])
+        networks[label]['floating_ips'].extend(net['floating_ips'])
+    return networks
+
+
 def get_networks_for_instance(context, instance):
     """Returns a prepared nw_info list for passing into the view
     builders
@@ -298,43 +356,14 @@ def get_networks_for_instance(context, instance):
      ...}
     """
 
-    def _emit_addr(ip, version):
-        return {'addr': ip, 'version': version}
-
-    networks = {}
-    fixed_ips = instance['fixed_ips']
-    ipv6_addrs_seen = {}
-    for fixed_ip in fixed_ips:
-        fixed_addr = fixed_ip['address']
-        network = fixed_ip['network']
-        vif = fixed_ip.get('virtual_interface')
-        if not network or not vif:
-            name = instance['name']
-            ip = fixed_ip['address']
-            LOG.warn(_("Instance %(name)s has stale IP "
-                    "address: %(ip)s (no network or vif)") % locals())
-            continue
-        label = network.get('label', None)
-        if label is None:
-            continue
-        if label not in networks:
-            networks[label] = {'ips': [], 'floating_ips': []}
-        nw_dict = networks[label]
-        cidr_v6 = network.get('cidr_v6')
-        if FLAGS.use_ipv6 and cidr_v6:
-            ipv6_addr = ipv6.to_global(cidr_v6, vif['address'],
-                    network['project_id'])
-            # Only add same IPv6 address once.  It's possible we've
-            # seen it before if there was a previous fixed_ip with
-            # same network and vif as this one
-            if not ipv6_addrs_seen.get(ipv6_addr):
-                nw_dict['ips'].append(_emit_addr(ipv6_addr, 6))
-                ipv6_addrs_seen[ipv6_addr] = True
-        nw_dict['ips'].append(_emit_addr(fixed_addr, 4))
-        for floating_ip in fixed_ip.get('floating_ips', []):
-            float_addr = floating_ip['address']
-            nw_dict['floating_ips'].append(_emit_addr(float_addr, 4))
-    return networks
+    try:
+        return get_networks_for_instance_from_cache(instance)
+    except (ValueError, KeyError, AttributeError):
+        # NOTE(jkoelker) If the json load (ValueError) or the
+        #                sqlalchemy FK (KeyError, AttributeError)
+        #                fail fall back to calling out the the
+        #                network api
+        return get_networks_for_instance_from_nwinfo(context, instance)
 
 
 def raise_http_conflict_for_instance_invalid_state(exc, action):
diff --git a/nova/api/openstack/compute/contrib/floating_ips.py b/nova/api/openstack/compute/contrib/floating_ips.py
index 2400f6c83..a7e29d5fb 100644
--- a/nova/api/openstack/compute/contrib/floating_ips.py
+++ b/nova/api/openstack/compute/contrib/floating_ips.py
@@ -68,7 +68,7 @@ def _translate_floating_ip_view(floating_ip):
     except (TypeError, KeyError):
         result['fixed_ip'] = None
     try:
-        result['instance_id'] = floating_ip['fixed_ip']['instance']['uuid']
+        result['instance_id'] = floating_ip['instance']['uuid']
     except (TypeError, KeyError):
         result['instance_id'] = None
     return {'floating_ip': result}
@@ -83,9 +83,35 @@ class FloatingIPController(object):
     """The Floating IPs API controller for the OpenStack API."""
 
     def __init__(self):
+        self.compute_api = compute.API()
         self.network_api = network.API()
         super(FloatingIPController, self).__init__()
 
+    def _get_fixed_ip(self, context, fixed_ip_id):
+        if fixed_ip_id is None:
+            return None
+        try:
+            return self.network_api.get_fixed_ip(context, fixed_ip_id)
+        except exception.FixedIpNotFound:
+            return None
+
+    def _get_instance(self, context, instance_id):
+        return self.compute_api.get(context, instance_id)
+
+    def _set_metadata(self, context, floating_ip):
+        fixed_ip_id = floating_ip['fixed_ip_id']
+        floating_ip['fixed_ip'] = self._get_fixed_ip(context,
+                                                     fixed_ip_id)
+        instance_id = None
+        if floating_ip['fixed_ip']:
+            instance_id = floating_ip['fixed_ip']['instance_id']
+
+        if instance_id:
+            floating_ip['instance'] = self._get_instance(context,
+                                                         instance_id)
+        else:
+            floating_ip['instance'] = None
+
     @wsgi.serializers(xml=FloatingIPTemplate)
     def show(self, req, id):
         """Return data about the given floating ip."""
@@ -96,6 +122,8 @@ class FloatingIPController(object):
         except exception.NotFound:
             raise webob.exc.HTTPNotFound()
 
+        self._set_metadata(context, floating_ip)
+
         return _translate_floating_ip_view(floating_ip)
 
     @wsgi.serializers(xml=FloatingIPsTemplate)
@@ -105,6 +133,9 @@ class FloatingIPController(object):
 
         floating_ips = self.network_api.get_floating_ips_by_project(context)
 
+        for floating_ip in floating_ips:
+            self._set_metadata(context, floating_ip)
+
         return _translate_floating_ips_view(floating_ips)
 
     @wsgi.serializers(xml=FloatingIPTemplate)
@@ -134,7 +165,7 @@ class FloatingIPController(object):
         context = req.environ['nova.context']
         floating_ip = self.network_api.get_floating_ip(context, id)
 
-        if floating_ip.get('fixed_ip'):
+        if floating_ip.get('fixed_ip_id'):
             self.network_api.disassociate_floating_ip(context,
                                                       floating_ip['address'])
 
@@ -207,7 +238,7 @@ class Floating_ips(extensions.ExtensionDescriptor):
 
         floating_ip = self.network_api.get_floating_ip_by_address(context,
                                                                   address)
-        if floating_ip.get('fixed_ip'):
+        if floating_ip.get('fixed_ip_id'):
             try:
                 self.network_api.disassociate_floating_ip(context, address)
             except exception.NotAuthorized, e:
diff --git a/nova/api/openstack/compute/views/addresses.py b/nova/api/openstack/compute/views/addresses.py
index 776ba9e59..5a633a10c 100644
--- a/nova/api/openstack/compute/views/addresses.py
+++ b/nova/api/openstack/compute/views/addresses.py
@@ -35,7 +35,7 @@ class ViewBuilder(common.ViewBuilder):
         """Return a dictionary describing an IP address."""
         return {
             "version": ip["version"],
-            "addr": ip["addr"],
+            "addr": ip["address"],
         }
 
     def show(self, network, label):