fixing more network issues

7 files changed, 141 insertions, 77 deletions

diff --git a/nova/auth/manager.py b/nova/auth/manager.py
index 69816882e..eed67d8c3 100644
--- a/nova/auth/manager.py
+++ b/nova/auth/manager.py
@@ -529,11 +529,9 @@ class AuthManager(object):
                                                member_users)
             if project_dict:
                 project = Project(**project_dict)
-                # FIXME(ja): EVIL HACK - this should poll from a pool
-                session = models.create_session()
-                net = models.Network(project_id=project.id, kind='vlan')
-                session.add(net)
-                session.commit()
+                # FIXME(ja): EVIL HACK
+                net = models.Network(project_id=project.id)
+                net.save()
                 return project
 
     def add_to_project(self, user, project):
@@ -580,6 +578,10 @@ class AuthManager(object):
 
     def delete_project(self, project):
         """Deletes a project"""
+        # FIXME(ja): EVIL HACK
+        if not isinstance(project, Project):
+            project = self.get_project(project)
+        project.network.delete()
         with self.driver() as drv:
             return drv.delete_project(Project.safe_id(project))
 
@@ -714,15 +716,15 @@ class AuthManager(object):
         zippy.writestr(FLAGS.credential_key_file, private_key)
         zippy.writestr(FLAGS.credential_cert_file, signed_cert)
 
-        network_data = vpn.NetworkData.lookup(pid)
-        if network_data:
+        (vpn_ip, vpn_port) = self.get_project_vpn_data(project)
+        if vpn_ip:
             configfile = open(FLAGS.vpn_client_template,"r")
             s = string.Template(configfile.read())
             configfile.close()
             config = s.substitute(keyfile=FLAGS.credential_key_file,
                                   certfile=FLAGS.credential_cert_file,
-                                  ip=network_data.ip,
-                                  port=network_data.port)
+                                  ip=vpn_ip,
+                                  port=vpn_port)
             zippy.writestr(FLAGS.credential_vpn_file, config)
         else:
             logging.warn("No vpn data for project %s" %
diff --git a/nova/models.py b/nova/models.py
index aa9f3da09..70010eab1 100644
--- a/nova/models.py
+++ b/nova/models.py
@@ -65,19 +65,24 @@ class NovaBase(object):
     @classmethod
     def all(cls):
         session = NovaBase.get_session()
-        return session.query(cls).all()
+        result = session.query(cls).all()
+        session.commit()
+        return result
 
     @classmethod
     def count(cls):
         session = NovaBase.get_session()
-        return session.query(cls).count()
+        result = session.query(cls).count()
+        session.commit()
+        return result
 
     @classmethod
     def find(cls, obj_id):
         session = NovaBase.get_session()
-        #print cls
         try:
-            return session.query(cls).filter_by(id=obj_id).one()
+            result = session.query(cls).filter_by(id=obj_id).one()
+            session.commit()
+            return result
         except exc.NoResultFound:
             raise exception.NotFound("No model for id %s" % obj_id)
 
@@ -89,12 +94,13 @@ class NovaBase(object):
     def delete(self):
         session = NovaBase.get_session()
         session.delete(self)
-        session.flush()
+        session.commit()
 
     def refresh(self):
         session = NovaBase.get_session()
         session.refresh(self)
 
+
 class Image(Base, NovaBase):
     __tablename__ = 'images'
     id = Column(Integer, primary_key=True)
@@ -128,9 +134,29 @@ class Image(Base, NovaBase):
             assert(val is None)
 
 
-class PhysicalNode(Base):
+class PhysicalNode(Base, NovaBase):
     __tablename__ = 'physical_nodes'
+    id = Column(String(255), primary_key=True)
+
+class Daemon(Base, NovaBase):
+    __tablename__ = 'daemons'
     id = Column(Integer, primary_key=True)
+    node_name = Column(String(255))  #, ForeignKey('physical_node.id'))
+    binary = Column(String(255))
+    report_count = Column(Integer)
+
+    @classmethod
+    def find_by_args(cls, node_name, binary):
+        session = NovaBase.get_session()
+        try:
+            query = session.query(cls).filter_by(node_name=node_name)
+            result = query.filter_by(binary=binary).one()
+            session.commit()
+            return result
+        except exc.NoResultFound:
+            raise exception.NotFound("No model for %s, %s" % (node_name,
+                                                              binary))
+
 
 class Instance(Base, NovaBase):
     __tablename__ = 'instances'
@@ -153,7 +179,7 @@ class Instance(Base, NovaBase):
         return "i-%s" % self.id
 
 
-    image_id = Column(Integer, ForeignKey('images.id'), nullable=False)
+    image_id = Column(Integer, ForeignKey('images.id'), nullable=True)
     kernel_id = Column(Integer, ForeignKey('images.id'), nullable=True)
     ramdisk_id = Column(Integer, ForeignKey('images.id'), nullable=True)
 
@@ -204,8 +230,7 @@ class Volume(Base, NovaBase):
     user_id = Column(String(255)) #, ForeignKey('users.id'), nullable=False)
     project_id = Column(String(255)) #, ForeignKey('projects.id'))
 
-    # FIXME: should be physical_node_id = Column(Integer)
-    node_name = Column(String(255))
+    node_name = Column(String(255))  #, ForeignKey('physical_node.id'))
     size = Column(Integer)
     alvailability_zone = Column(String(255)) # FIXME foreign key?
     instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True)
@@ -223,37 +248,6 @@ class ExportDevice(Base, NovaBase):
     volume = relationship(Volume, backref=backref('export_device',
                                                   uselist=False))
 
-class Network(Base, NovaBase):
-    __tablename__ = 'networks'
-    id = Column(Integer, primary_key=True)
-    kind = Column(String(255))
-
-    injected = Column(Boolean, default=False)
-    network_str = Column(String(255))
-    netmask = Column(String(255))
-    bridge = Column(String(255))
-    gateway = Column(String(255))
-    broadcast = Column(String(255))
-    dns = Column(String(255))
-
-    vlan = Column(Integer)
-    vpn_public_ip_str = Column(String(255))
-    vpn_public_port = Column(Integer)
-    vpn_private_ip_str = Column(String(255))
-
-    project_id = Column(String(255)) #, ForeignKey('projects.id'), nullable=False)
-    # FIXME: should be physical_node_id = Column(Integer)
-    node_name = Column(String(255))
-
-
-class NetworkIndex(Base, NovaBase):
-    __tablename__ = 'network_indexes'
-    id = Column(Integer, primary_key=True)
-    index = Column(Integer)
-    network_id = Column(Integer, ForeignKey('networks.id'), nullable=True)
-    network = relationship(Network, backref=backref('network_index',
-                                                      uselist=False))
-
 
 #FIXME can these both come from the same baseclass?
 class FixedIp(Base, NovaBase):
@@ -261,7 +255,6 @@ class FixedIp(Base, NovaBase):
     id = Column(Integer, primary_key=True)
     ip_str = Column(String(255), unique=True)
     network_id = Column(Integer, ForeignKey('networks.id'), nullable=False)
-    network = relationship(Network, backref=backref('fixed_ips'))
     instance_id = Column(Integer, ForeignKey('instances.id'), nullable=True)
     instance = relationship(Instance, backref=backref('fixed_ip',
                                                       uselist=False))
@@ -273,10 +266,13 @@ class FixedIp(Base, NovaBase):
     def find_by_ip_str(cls, ip_str):
         session = NovaBase.get_session()
         try:
-            return session.query(cls).filter_by(ip_str=ip_str).one()
+            result = session.query(cls).filter_by(ip_str=ip_str).one()
+            session.commit()
+            return result
         except exc.NoResultFound:
             raise exception.NotFound("No model for ip str %s" % ip_str)
 
+
 class ElasticIp(Base, NovaBase):
     __tablename__ = 'elastic_ips'
     id = Column(Integer, primary_key=True)
@@ -285,18 +281,57 @@ class ElasticIp(Base, NovaBase):
     fixed_ip = relationship(FixedIp, backref=backref('elastic_ips'))
 
     project_id = Column(String(255)) #, ForeignKey('projects.id'), nullable=False)
-    # FIXME: should be physical_node_id = Column(Integer)
-    node_name = Column(String(255))
+    node_name = Column(String(255))  #, ForeignKey('physical_node.id'))
 
     @classmethod
     def find_by_ip_str(cls, ip_str):
         session = NovaBase.get_session()
         try:
-            return session.query(cls).filter_by(ip_str=ip_str).one()
+            result = session.query(cls).filter_by(ip_str=ip_str).one()
+            session.commit()
+            return result
         except exc.NoResultFound:
             raise exception.NotFound("No model for ip str %s" % ip_str)
 
 
+class Network(Base, NovaBase):
+    __tablename__ = 'networks'
+    id = Column(Integer, primary_key=True)
+    kind = Column(String(255))
+
+    injected = Column(Boolean, default=False)
+    network_str = Column(String(255))
+    netmask = Column(String(255))
+    bridge = Column(String(255))
+    gateway = Column(String(255))
+    broadcast = Column(String(255))
+    dns = Column(String(255))
+
+    vlan = Column(Integer)
+    vpn_public_ip_str = Column(String(255))
+    vpn_public_port = Column(Integer)
+    vpn_private_ip_str = Column(String(255))
+
+    project_id = Column(String(255)) #, ForeignKey('projects.id'), nullable=False)
+    node_name = Column(String(255))  #, ForeignKey('physical_node.id'))
+
+    fixed_ips = relationship(FixedIp,
+                             single_parent=True,
+                             backref=backref('network'),
+                             cascade='all, delete, delete-orphan')
+
+
+class NetworkIndex(Base, NovaBase):
+    __tablename__ = 'network_indexes'
+    id = Column(Integer, primary_key=True)
+    index = Column(Integer)
+    network_id = Column(Integer, ForeignKey('networks.id'), nullable=True)
+    network = relationship(Network, backref=backref('network_index',
+                                                      uselist=False))
+
+
+
+
 def create_session(engine=None):
     return NovaBase.get_session()
 
diff --git a/nova/network/service.py b/nova/network/service.py
index 938d7832b..45bcf58ad 100644
--- a/nova/network/service.py
+++ b/nova/network/service.py
@@ -118,12 +118,14 @@ class BaseNetworkService(service.Service):
         # NOTE(vish): if with_lockmode isn't supported, as in sqlite,
         #             then this has concurrency issues
         if network.node_name:
+            session.commit()
             return network.node_name
         network.node_name = FLAGS.node_name
         network.kind = FLAGS.network_type
         session.add(network)
         session.commit()
         self._on_set_network_host(network)
+        return network.node_name
 
     def allocate_fixed_ip(self, project_id, instance_id, *args, **kwargs):
         """Gets fixed ip from the pool"""
@@ -132,7 +134,8 @@ class BaseNetworkService(service.Service):
         session = models.NovaBase.get_session()
         query = session.query(models.FixedIp).filter_by(network_id=network.id)
         query = query.filter_by(reserved=False).filter_by(allocated=False)
-        fixed_ip = query.filter_by(leased=False).with_lockmode("update").first
+        query = query.filter_by(leased=False).with_lockmode("update")
+        fixed_ip = query.first()
         # NOTE(vish): if with_lockmode isn't supported, as in sqlite,
         #             then this has concurrency issues
         if not fixed_ip:
@@ -233,16 +236,19 @@ class VlanNetworkService(BaseNetworkService):
         # NOTE(vish): this should probably be removed and added via
         #             admin command or fixtures
         if models.NetworkIndex.count() == 0:
+            session = models.NovaBase.get_session()
             for i in range(FLAGS.num_networks):
                 network_index = models.NetworkIndex()
                 network_index.index = i
-                network_index.save()
+                session.add(network_index)
+            session.commit()
 
     def allocate_fixed_ip(self, project_id, instance_id,  is_vpn=False,
                           *args, **kwargs):
         """Gets a fixed ip from the pool"""
         network = get_network_for_project(project_id)
         if is_vpn:
+            # FIXME concurrency issue?
             fixed_ip = models.FixedIp.find_by_ip_str(network.vpn_private_ip_str)
             if fixed_ip.allocated:
                 raise network_exception.AddressAlreadyAllocated()
@@ -258,7 +264,6 @@ class VlanNetworkService(BaseNetworkService):
         else:
             parent = super(VlanNetworkService, self)
             ip_str = parent.allocate_fixed_ip(project_id, instance_id)
-        logging.debug("sql %s", FLAGS.sql_connection)
         _driver.ensure_vlan_bridge(network.vlan, network.bridge)
         return ip_str
 
@@ -275,13 +280,16 @@ class VlanNetworkService(BaseNetworkService):
 
     def lease_ip(self, fixed_ip_str):
         """Called by bridge when ip is leased"""
-        logging.debug("sql %s", FLAGS.sql_connection)
         fixed_ip = models.FixedIp.find_by_ip_str(fixed_ip_str)
         if not fixed_ip.allocated:
             raise network_exception.AddressNotAllocated(fixed_ip_str)
         logging.debug("Leasing IP %s", fixed_ip_str)
         fixed_ip.leased = True
         fixed_ip.save()
+        print fixed_ip.allocated
+        print fixed_ip.leased
+        print fixed_ip.instance_id
+        print 'ip %s leased' % fixed_ip_str
 
     def release_ip(self, fixed_ip_str):
         """Called by bridge when ip is released"""
@@ -321,13 +329,15 @@ class VlanNetworkService(BaseNetworkService):
         BOTTOM_RESERVED = 3
         TOP_RESERVED = 1 + FLAGS.cnt_vpn_clients
         num_ips = len(project_net)
+        session = models.NovaBase.get_session()
         for i in range(num_ips):
             fixed_ip = models.FixedIp()
             fixed_ip.ip_str = str(project_net[i])
             if i < BOTTOM_RESERVED or num_ips - i < TOP_RESERVED:
                 fixed_ip.reserved = True
             fixed_ip.network = network
-            fixed_ip.save()
+            session.add(fixed_ip)
+        session.commit()
 
 
     def _get_network_index(self, network):
diff --git a/nova/service.py b/nova/service.py
index 96281bc6b..4c35bdefa 100644
--- a/nova/service.py
+++ b/nova/service.py
@@ -28,10 +28,10 @@ from twisted.internet import defer
 from twisted.internet import task
 from twisted.application import service
 
-from nova import datastore
+from nova import exception
 from nova import flags
+from nova import models
 from nova import rpc
-from nova.compute import model
 
 
 FLAGS = flags.FLAGS
@@ -87,17 +87,28 @@ class Service(object, service.Service):
         return application
 
     @defer.inlineCallbacks
-    def report_state(self, nodename, daemon):
+    def report_state(self, node_name, binary):
+        """Update the state of this daemon in the datastore"""
         # TODO(termie): make this pattern be more elegant. -todd
         try:
-            record = model.Daemon(nodename, daemon)
-            record.heartbeat()
+            try:
+                #FIXME abstract this
+                daemon = models.find_by_args(node_name, binary)
+            except exception.NotFound():
+                daemon = models.Daemon(node_name=node_name,
+                                       binary=binary)
+            self._update_daemon()
+            self.commit()
             if getattr(self, "model_disconnected", False):
                 self.model_disconnected = False
                 logging.error("Recovered model server connection!")
 
-        except datastore.ConnectionError, ex:
+        except Exception, ex: #FIXME this should only be connection error
             if not getattr(self, "model_disconnected", False):
                 self.model_disconnected = True
                 logging.exception("model server went away")
         yield
+
+    def _update_daemon(daemon):
+        """Set any extra daemon data here"""
+        daemon.report_count = daemon.report_count + 1
diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py
index 0b404bfdc..59a81818c 100644
--- a/nova/tests/auth_unittest.py
+++ b/nova/tests/auth_unittest.py
@@ -32,7 +32,6 @@ FLAGS = flags.FLAGS
 
 
 class AuthTestCase(test.BaseTestCase):
-    flush_db = False
     def setUp(self):
         super(AuthTestCase, self).setUp()
         self.flags(connection_type='fake',
diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py
index 00aaac346..c94c81f72 100644
--- a/nova/tests/network_unittest.py
+++ b/nova/tests/network_unittest.py
@@ -23,6 +23,7 @@ import os
 import logging
 import tempfile
 
+from nova import exception
 from nova import flags
 from nova import models
 from nova import test
@@ -40,10 +41,10 @@ class NetworkTestCase(test.TrialTestCase):
         super(NetworkTestCase, self).setUp()
         # NOTE(vish): if you change these flags, make sure to change the
         #             flags in the corresponding section in nova-dhcpbridge
-        fd, sqlfile = tempfile.mkstemp()
-        self.sqlfile = os.path.abspath(sqlfile)
+        self.sqlfile = 'test.sqlite'
         self.flags(connection_type='fake',
-                   sql_connection='sqlite:///%s' % self.sqlfile,
+                   #sql_connection='sqlite:///%s' % self.sqlfile,
+                   sql_connection='mysql://root@localhost/test',
                    fake_storage=True,
                    fake_network=True,
                    auth_driver='nova.auth.ldapdriver.FakeLdapDriver',
@@ -53,6 +54,7 @@ class NetworkTestCase(test.TrialTestCase):
         self.manager = manager.AuthManager()
         self.user = self.manager.create_user('netuser', 'netuser', 'netuser')
         self.projects = []
+        print FLAGS.sql_connection
         self.service = service.VlanNetworkService()
         for i in range(5):
             name = 'project%s' % i
@@ -64,7 +66,6 @@ class NetworkTestCase(test.TrialTestCase):
         instance = models.Instance()
         instance.mac_address = utils.generate_mac()
         instance.hostname = 'fake'
-        instance.image_id = 'fake'
         instance.save()
         self.instance_id = instance.id
 
@@ -73,16 +74,19 @@ class NetworkTestCase(test.TrialTestCase):
         for project in self.projects:
             self.manager.delete_project(project)
         self.manager.delete_user(self.user)
-        os.unlink(self.sqlfile)
 
     def test_public_network_association(self):
         """Makes sure that we can allocaate a public ip"""
         # FIXME better way of adding elastic ips
         pubnet = IPy.IP(flags.FLAGS.public_range)
-        elastic_ip = models.ElasticIp()
-        elastic_ip.ip_str = str(pubnet[0])
-        elastic_ip.node_name = FLAGS.node_name
-        elastic_ip.save()
+        ip_str = str(pubnet[0])
+        try:
+            elastic_ip = models.ElasticIp.find_by_ip_str(ip_str)
+        except exception.NotFound:
+            elastic_ip = models.ElasticIp()
+            elastic_ip.ip_str = ip_str
+            elastic_ip.node_name = FLAGS.node_name
+            elastic_ip.save()
         eaddress = self.service.allocate_elastic_ip(self.projects[0].id)
         faddress = self.service.allocate_fixed_ip(self.projects[0].id,
                                                  self.instance_id)
@@ -101,7 +105,11 @@ class NetworkTestCase(test.TrialTestCase):
                                                  self.instance_id)
         net = service.get_network_for_project(self.projects[0].id)
         self.assertTrue(is_allocated_in_project(address, self.projects[0].id))
+        print 'I just got allocated'
         issue_ip(address, net.bridge, self.sqlfile)
+        obj = models.FixedIp.find_by_ip_str(address)
+        obj.refresh()
+        print obj.leased
         self.service.deallocate_fixed_ip(address)
 
         # Doesn't go away until it's dhcp released
@@ -178,7 +186,7 @@ class NetworkTestCase(test.TrialTestCase):
     def test_too_many_networks(self):
         """Ensure error is raised if we run out of vpn ports"""
         projects = []
-        networks_left = FLAGS.num_networks - len(self.projects)
+        networks_left = FLAGS.num_networks - models.Network.count()
         for i in range(networks_left):
             project = self.manager.create_project('many%s' % i, self.user)
             self.service.set_network_host(project.id)
diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py
index 62ea2a26c..82f71901a 100644
--- a/nova/tests/volume_unittest.py
+++ b/nova/tests/volume_unittest.py
@@ -86,7 +86,6 @@ class VolumeTestCase(test.TrialTestCase):
         for i in xrange(self.total_slots):
             vid = yield self.volume.create_volume(vol_size, user_id, project_id)
             vols.append(vid)
-            print models.Volume.find(vid).export_device.volume_id
         self.assertFailure(self.volume.create_volume(vol_size,
                                                      user_id,
                                                      project_id),