A few more changes:

 * Fixed up some flags
 * Put in an updated nova.sh
 * Broke out metadata forwarding so it will work in flatdhcp mode
 * Added descriptive docstrings explaining the networking modes in more detail

Original Message:

FlatDHCPManager wasn't quite working.  This makes it work again.  I discovered that automatically bridging into eth0 without a vlan is very dangerous.  If eth0 has an ip that you are using for ssh, you lose your connectivity.  For that reason, I set bridge_dev to None by default.  This unfortunately means that you have to specify a bridge_dev for VlanManager to actually work.

3 files changed, 114 insertions, 35 deletions

diff --git a/contrib/nova.sh b/contrib/nova.sh
index f47d972ab..6033c9e3a 100755
--- a/contrib/nova.sh
+++ b/contrib/nova.sh
@@ -17,11 +17,17 @@ if [ ! -n "$HOST_IP" ]; then
     #             you should explicitly set HOST_IP in your environment
     HOST_IP=`ifconfig  | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'`
 fi
-TEST=0
-USE_MYSQL=0
-MYSQL_PASS=nova
-USE_LDAP=0
-LIBVIRT_TYPE=qemu
+
+USE_MYSQL=${USE_MYSQL:-0}
+MYSQL_PASS=${MYSQL_PASS:-nova}
+TEST=${TEST:-0}
+USE_LDAP=${USE_LDAP:-0}
+LIBVIRT_TYPE=${LIBVIRT_TYPE:-qemu}
+NET_MAN=${NET_MAN:-FlatDHCPManager}
+# NOTE(vish): If you are using FlatDHCP on multiple hosts, set the interface
+#             below but make sure that the interface doesn't already have an
+#             ip or you risk breaking things.
+# FLAT_INTERFACE=eth0
 
 if [ "$USE_MYSQL" == 1 ]; then
     SQL_CONN=mysql://root:$MYSQL_PASS@localhost/nova
@@ -40,6 +46,8 @@ cat >/etc/nova/nova-manage.conf << NOVA_CONF_EOF
 --verbose
 --nodaemon
 --dhcpbridge_flagfile=/etc/nova/nova-manage.conf
+--FAKE_subdomain=ec2
+--network_manager=nova.network.manager.$NET_MAN
 --cc_host=$HOST_IP
 --routing_source_ip=$HOST_IP
 --sql_connection=$SQL_CONN
@@ -47,6 +55,10 @@ cat >/etc/nova/nova-manage.conf << NOVA_CONF_EOF
 --libvirt_type=$LIBVIRT_TYPE
 NOVA_CONF_EOF
 
+if [ -n "$FLAT_INTERFACE" ]; then
+    echo "--flat_interface=$FLAT_INTERFACE" >>/etc/nova/nova-manage.conf
+fi
+
 if [ "$CMD" == "branch" ]; then
     sudo apt-get install -y bzr
     rm -rf $NOVA_DIR
@@ -61,9 +73,12 @@ if [ "$CMD" == "install" ]; then
     sudo apt-get install -y python-software-properties
     sudo add-apt-repository ppa:nova-core/ppa
     sudo apt-get update
-    sudo apt-get install -y dnsmasq open-iscsi kpartx kvm gawk iptables ebtables
+    sudo apt-get install -y dnsmasq kpartx kvm gawk iptables ebtables
     sudo apt-get install -y user-mode-linux kvm libvirt-bin
-    sudo apt-get install -y screen iscsitarget euca2ools vlan curl rabbitmq-server
+    sudo apt-get install -y screen euca2ools vlan curl rabbitmq-server
+    sudo apt-get install -y lvm2 iscsitarget open-iscsi
+    echo "ISCSITARGET_ENABLE=true" | sudo tee /etc/default/iscsitarget
+    sudo /etc/init.d/iscsitarget restart
     sudo modprobe kvm
     sudo /etc/init.d/libvirt-bin restart
     sudo apt-get install -y python-twisted python-sqlalchemy python-mox python-greenlet python-carrot
@@ -122,8 +137,8 @@ if [ "$CMD" == "run" ]; then
     $NOVA_DIR/bin/nova-manage project create admin admin
     # export environment variables for project 'admin' and user 'admin'
     $NOVA_DIR/bin/nova-manage project environment admin admin $NOVA_DIR/novarc
-    # create 3 small networks
-    $NOVA_DIR/bin/nova-manage network create 10.0.0.0/8 3 16
+    # create a small network
+    $NOVA_DIR/bin/nova-manage network create 10.0.0.0/8 1 32
 
     # nova api crashes if we start it with a regular screen command,
     # so send the start command by forcing text into the window.
@@ -134,19 +149,20 @@ if [ "$CMD" == "run" ]; then
     screen_it scheduler "$NOVA_DIR/bin/nova-scheduler --flagfile=/etc/nova/nova-manage.conf"
     screen_it volume "$NOVA_DIR/bin/nova-volume --flagfile=/etc/nova/nova-manage.conf"
     screen_it test ". $NOVA_DIR/novarc"
-    screen -x
+    screen -S nova -x
 fi
 
 if [ "$CMD" == "run" ] || [ "$CMD" == "terminate" ]; then
     # shutdown instances
     . $NOVA_DIR/novarc; euca-describe-instances | grep i- | cut -f2 | xargs euca-terminate-instances
     sleep 2
+    # delete volumes
+    . $NOVA_DIR/novarc; euca-describe-volumes | grep vol- | cut -f2 | xargs -n1 euca-delete-volume
 fi
 
 if [ "$CMD" == "run" ] || [ "$CMD" == "clean" ]; then
     screen -S nova -X quit
     rm *.pid*
-    $NOVA_DIR/tools/setup_iptables.sh clear
 fi
 
 if [ "$CMD" == "scrub" ]; then
diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py
index 4ea24cda6..391abfb76 100644
--- a/nova/network/linux_net.py
+++ b/nova/network/linux_net.py
@@ -42,8 +42,8 @@ flags.DEFINE_string('networks_path', utils.abspath('../networks'),
                     'Location to keep network config files')
 flags.DEFINE_string('public_interface', 'vlan1',
                     'Interface for public IP addresses')
-flags.DEFINE_string('bridge_dev', 'eth0',
-                        'network device for bridges')
+flags.DEFINE_string('vlan_interface', 'eth0',
+                    'network device for vlans')
 flags.DEFINE_string('dhcpbridge', _bin_file('nova-dhcpbridge'),
                         'location of nova-dhcpbridge')
 flags.DEFINE_string('routing_source_ip', '127.0.0.1',
@@ -54,14 +54,15 @@ flags.DEFINE_bool('use_nova_chains', False,
 DEFAULT_PORTS = [("tcp", 80), ("tcp", 22), ("udp", 1194), ("tcp", 443)]
 
 
-def init_host():
-    """Basic networking setup goes here"""
-    # NOTE(devcamcar): Cloud public DNAT entries, CloudPipe port
-    # forwarding entries and a default DNAT entry.
+def metadata_forward():
+    """Create forwarding rule for metadata"""
     _confirm_rule("PREROUTING", "-t nat -s 0.0.0.0/0 "
              "-d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT "
              "--to-destination %s:%s" % (FLAGS.cc_host, FLAGS.cc_port))
 
+
+def init_host():
+    """Basic networking setup goes here"""
     # NOTE(devcamcar): Cloud public SNAT entries and the default
     # SNAT rule for outbound traffic.
     _confirm_rule("POSTROUTING", "-t nat -s %s "
@@ -134,7 +135,7 @@ def ensure_vlan(vlan_num):
     if not _device_exists(interface):
         logging.debug("Starting VLAN inteface %s", interface)
         _execute("sudo vconfig set_name_type VLAN_PLUS_VID_NO_PAD")
-        _execute("sudo vconfig add %s %s" % (FLAGS.bridge_dev, vlan_num))
+        _execute("sudo vconfig add %s %s" % (FLAGS.vlan_interface, vlan_num))
         _execute("sudo ifconfig %s up" % interface)
     return interface
 
@@ -142,12 +143,13 @@ def ensure_vlan(vlan_num):
 def ensure_bridge(bridge, interface, net_attrs=None):
     """Create a bridge unless it already exists"""
     if not _device_exists(bridge):
-        logging.debug("Starting Bridge inteface for %s", interface)
+        logging.debug("Starting Bridge interface for %s", interface)
         _execute("sudo brctl addbr %s" % bridge)
         _execute("sudo brctl setfd %s 0" % bridge)
         # _execute("sudo brctl setageing %s 10" % bridge)
         _execute("sudo brctl stp %s off" % bridge)
-        _execute("sudo brctl addif %s %s" % (bridge, interface))
+        if interface:
+            _execute("sudo brctl addif %s %s" % (bridge, interface))
     if net_attrs:
         _execute("sudo ifconfig %s %s broadcast %s netmask %s up" % \
                 (bridge,
diff --git a/nova/network/manager.py b/nova/network/manager.py
index b033bb0a4..a7298b47f 100644
--- a/nova/network/manager.py
+++ b/nova/network/manager.py
@@ -27,6 +27,7 @@ topologies.  All of the network commands are issued to a subclass of
 
 :network_driver:  Driver to use for network creation
 :flat_network_bridge:  Bridge device for simple network instances
+:flat_interface:  FlatDhcp will bridge into this interface if set
 :flat_network_dns:  Dns for simple network
 :flat_network_dhcp_start:  Dhcp start for FlatDhcp
 :vlan_start:  First VLAN for private networks
@@ -63,7 +64,11 @@ flags.DEFINE_string('flat_network_bridge', 'br100',
                     'Bridge for simple network instances')
 flags.DEFINE_string('flat_network_dns', '8.8.4.4',
                     'Dns for simple network')
-flags.DEFINE_string('flat_network_dhcp_start', '192.168.0.2',
+flags.DEFINE_bool('flat_injected', True,
+                  'Whether to attempt to inject network setup into guest')
+flags.DEFINE_string('flat_interface', None,
+                    'FlatDhcp will bridge into this interface if set')
+flags.DEFINE_string('flat_network_dhcp_start', '10.0.0.2',
                     'Dhcp start for FlatDhcp')
 flags.DEFINE_integer('vlan_start', 100, 'First VLAN for private networks')
 flags.DEFINE_integer('num_networks', 1000, 'Number of networks to support')
@@ -175,9 +180,11 @@ class NetworkManager(manager.Manager):
         if instance_ref['mac_address'] != mac:
             raise exception.Error("IP %s leased to bad mac %s vs %s" %
                                   (address, instance_ref['mac_address'], mac))
+        now = datetime.datetime.utcnow()
         self.db.fixed_ip_update(context,
                                 fixed_ip_ref['address'],
-                                {'leased': True})
+                                {'leased': True,
+                                 'updated_at': now})
         if not fixed_ip_ref['allocated']:
             logging.warn("IP %s leased that was already deallocated", address)
 
@@ -246,7 +253,31 @@ class NetworkManager(manager.Manager):
 
 
 class FlatManager(NetworkManager):
-    """Basic network where no vlans are used."""
+    """Basic network where no vlans are used.
+
+    FlatManager does not do any bridge or vlan creation.  The user is
+    responsible for setting up whatever bridge is specified in
+    flat_network_bridge (br100 by default).  This bridge needs to be created
+    on all compute hosts.
+
+    The idea is to create a single network for the host with a command like:
+    nova-manage network create 192.168.0.0/24 1 256. Creating multiple
+    networks for for one manager is currently not supported, but could be
+    added by modifying allocate_fixed_ip and get_network to get the a network
+    with new logic instead of network_get_by_bridge. Arbitrary lists of
+    addresses in a single network can be accomplished with manual db editing.
+
+    If flat_injected is True, the compute host will attempt to inject network
+    config into the guest.  It attempts to modify /etc/network/interfaces and
+    currently only works on debian based systems. To support a wider range of
+    OSes, some other method may need to be devised to let the guest know which
+    ip it should be using so that it can configure itself. Perhaps an attached
+    disk or serial device with configuration info.
+
+    Metadata forwarding must be handled by the gateway, and since nova does
+    not do any setup in this mode, it must be done manually.  Requests to
+    169.254.169.254 port 80 will need to be forwarded to the api server.
+    """
 
     def allocate_fixed_ip(self, context, instance_id, *args, **kwargs):
         """Gets a fixed ip from the pool."""
@@ -285,6 +316,7 @@ class FlatManager(NetworkManager):
             cidr = "%s/%s" % (fixed_net[start], significant_bits)
             project_net = IPy.IP(cidr)
             net = {}
+            net['bridge'] = FLAGS.flat_network_bridge
             net['cidr'] = cidr
             net['netmask'] = str(project_net.netmask())
             net['gateway'] = str(project_net[1])
@@ -306,18 +338,36 @@ class FlatManager(NetworkManager):
     def _on_set_network_host(self, context, network_id):
         """Called when this host becomes the host for a network."""
         net = {}
-        net['injected'] = True
-        net['bridge'] = FLAGS.flat_network_bridge
+        net['injected'] = FLAGS.flat_injected
         net['dns'] = FLAGS.flat_network_dns
         self.db.network_update(context, network_id, net)
 
 
-class FlatDHCPManager(NetworkManager):
-    """Flat networking with dhcp."""
+class FlatDHCPManager(FlatManager):
+    """Flat networking with dhcp.
+
+    FlatDHCPManager will start up one dhcp server to give out addresses.
+    It never injects network settings into the guest. Otherwise it behaves
+    like FlatDHCPManager.
+    """
+
+    def init_host(self):
+        """Do any initialization that needs to be run if this is a
+        standalone service.
+        """
+        super(FlatDHCPManager, self).init_host()
+        self.driver.metadata_forward()
+
+    def setup_compute_network(self, context, instance_id):
+        """Sets up matching network for compute hosts."""
+        network_ref = db.network_get_by_instance(context, instance_id)
+        self.driver.ensure_bridge(network_ref['bridge'],
+                                  FLAGS.flat_interface,
+                                  network_ref)
 
     def setup_fixed_ip(self, context, address):
         """Setup dhcp for this network."""
-        network_ref = db.fixed_ip_get_by_address(context, address)
+        network_ref = db.fixed_ip_get_network(context, address)
         self.driver.update_dhcp(context, network_ref['id'])
 
     def deallocate_fixed_ip(self, context, address, *args, **kwargs):
@@ -326,18 +376,28 @@ class FlatDHCPManager(NetworkManager):
 
     def _on_set_network_host(self, context, network_id):
         """Called when this host becomes the host for a project."""
-        super(FlatDHCPManager, self)._on_set_network_host(context, network_id)
-        network_ref = self.db.network_get(context, network_id)
-        self.db.network_update(context,
-                               network_id,
-                               {'dhcp_start': FLAGS.flat_network_dhcp_start})
+        net = {}
+        net['dhcp_start'] = FLAGS.flat_network_dhcp_start
+        self.db.network_update(context, network_id, net)
+        network_ref = db.network_get(context, network_id)
         self.driver.ensure_bridge(network_ref['bridge'],
-                                  FLAGS.bridge_dev,
+                                  FLAGS.flat_interface,
                                   network_ref)
 
 
 class VlanManager(NetworkManager):
-    """Vlan network with dhcp."""
+    """Vlan network with dhcp.
+
+    VlanManager is the most complicated.  It will create a host-managed
+    vlan for each project.  Each project gets its own subnet.  The networks
+    and associated subnets are created with nova-manage using a command like:
+    nova-manage network create 10.0.0.0/8 3 16.  This will create 3 networks
+    of 16 addresses from the beginning of the 10.0.0.0 range.
+
+    A dhcp server is run for each subnet, so each project will have its own.
+    For this mode to be useful, each project will need a vpn to access the
+    instances in its subnet.
+    """
 
     @defer.inlineCallbacks
     def periodic_tasks(self, context=None):
@@ -357,6 +417,7 @@ class VlanManager(NetworkManager):
         standalone service.
         """
         super(VlanManager, self).init_host()
+        self.driver.metadata_forward()
         self.driver.init_host()
 
     def allocate_fixed_ip(self, context, instance_id, *args, **kwargs):