Merge remote branch 'origin/master'

25 files changed, 225 insertions, 35 deletions

diff --git a/README b/README
index 29c6af0e5..ab43d46d6 100644
--- a/README
+++ b/README
@@ -18,3 +18,4 @@ To hack at it: read HACKING
 
 To watch it: http://184.106.205.70:8010
  and again and again (This is buildbot again)
+aand again nd again
diff --git a/bin/nova-manage b/bin/nova-manage
index ffb626b24..12e4c9324 100755
--- a/bin/nova-manage
+++ b/bin/nova-manage
@@ -220,7 +220,7 @@ def methods_of(obj):
 
 
 if __name__ == '__main__':
-    utils.default_flagfile()
+    utils.default_flagfile('/etc/nova/nova-manage.conf')
     argv = FLAGS(sys.argv)
     script_name = argv.pop(0)
     if len(argv) < 1:
diff --git a/debian/changelog b/debian/changelog
index 0f648e30c..31dd5e91e 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,23 @@
+nova (0.2.3-1) UNRELEASED; urgency=low
+
+  * Relax the Twisted dependency to python-twisted-core (rather than the
+    full stack).
+  * Move nova related configuration files into /etc/nova/.
+  * Add a dependency on nginx from nova-objectsstore and install a
+    suitable configuration file.
+  * Ship the CA directory in nova-common.
+  * Add a default flag file for nova-manage to help it find the CA.
+  * If set, pass KernelId and RamdiskId from RunInstances call to the
+    target compute node.
+  * Added --network_path setting to nova-compute's flagfile.
+  * Move templates from python directories to /usr/share/nova.
+  * Add debian/nova-common.dirs to create
+    var/lib/nova/{buckets,CA,images,instances,keys,networks}
+  * Don't pass --daemonize=1 to nova-compute. It's already daemonising
+    by default.
+
+ -- Vishvananda Ishaya <vishvananda@gmail.com>  Mon, 14 Jul 2010 12:00:00 -0700
+
 nova (0.2.2-10) UNRELEASED; urgency=low
 
   * Fixed extra space in vblade-persist
diff --git a/debian/control b/debian/control
index bb2071234..342dfb185 100644
--- a/debian/control
+++ b/debian/control
@@ -3,36 +3,112 @@ Section: net
 Priority: extra
 Maintainer: Jesse Andrews <jesse@ansolabs.com>
 Build-Depends: debhelper (>= 7)
-Build-Depends-Indep: python-support
+Build-Depends-Indep: python-support, python-setuptools
 Standards-Version: 3.8.4
 XS-Python-Version: 2.6
 
 Package: nova-common
 Architecture: all
-Depends: ${python:Depends}, aoetools, vlan, python-ipy, python-boto, python-m2crypto, python-pycurl, python-twisted (>= 10.0.0-2ubuntu2nebula1), python-daemon, python-redis, python-carrot, python-lockfile, python-gflags, python-tornado, ${misc:Depends}
+Depends: ${python:Depends}, aoetools, vlan, python-ipy, python-boto, python-m2crypto, python-pycurl, python-twisted-core, python-daemon, python-redis, python-carrot, python-lockfile, python-gflags, python-tornado, ${misc:Depends}
 Provides: ${python:Provides}
-Conflicts: nova
-Description: Nova is a cloud
+Description: Nova Cloud Computing - common files
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This package contains things that are needed by all parts of Nova.
 
 Package: nova-compute
 Architecture: all
-Depends: nova-common (= ${binary:Version}), kpartx, kvm, python-libvirt, libvirt-bin (>= 0.8.1), ${python:Depends}, ${misc:Depends}
-Description: Nova compute
+Depends: nova-common (= ${binary:Version}), kpartx, kvm, python-libvirt, libvirt-bin (>= 0.7.5), curl, ${python:Depends}, ${misc:Depends}
+Description: Nova Cloud Computing - compute node
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This is the package you will install on the nodes that will run your
+ virtual machines.
 
 Package: nova-volume
 Architecture: all
 Depends: nova-common (= ${binary:Version}), vblade, vblade-persist, ${python:Depends}, ${misc:Depends}
-Description: Nova volume
+Description: Nova Cloud Computing - storage
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This is the package you will install on your storage nodes.
 
 Package: nova-api
 Architecture: all
 Depends: nova-common (= ${binary:Version}), ${python:Depends}, ${misc:Depends}
-Description: Nova api
+Description: Nova Cloud Computing - API frontend 
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This package provides the API frontend.
 
 Package: nova-objectstore
 Architecture: all
-Depends: nova-common (= ${binary:Version}), ${python:Depends}, ${misc:Depends}
-Description: Nova object store
+Depends: nova-common (= ${binary:Version}), nginx, ${python:Depends}, ${misc:Depends}
+Description: Nova Cloud Computing - object store
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This is the package you will install on the nodes that will contain your
+ object store.
 
 Package: nova-instancemonitor
 Architecture: all
@@ -42,4 +118,19 @@ Description: Nova instance monitor
 Package: nova-tools
 Architecture: all
 Depends: python-boto, ${python:Depends}, ${misc:Depends}
-Description: CLI tools to access nova
+Description: Nova Cloud Computing - management tools
+ Nova is a cloud computing fabric controller (the main part of an IaaS
+ system) built to match the popular AWS EC2 and S3 APIs. It is written in
+ Python, using the Tornado and Twisted frameworks, and relies on the
+ standard AMQP messaging protocol, and the Redis distributed KVS.
+ .
+ Nova is intended to be easy to extend, and adapt. For example, it
+ currently uses an LDAP server for users and groups, but also includes a
+ fake LDAP server, that stores data in Redis. It has extensive test
+ coverage, and uses the Sphinx toolkit (the same as Python itself) for code
+ and user documentation.
+ .
+ While Nova is currently in Beta use within several organizations, the
+ codebase is very much under active development.
+ .
+ This package contains admin tools for Nova.
diff --git a/debian/nova-api.conf b/debian/nova-api.conf
new file mode 100644
index 000000000..9cd4051b1
--- /dev/null
+++ b/debian/nova-api.conf
@@ -0,0 +1,5 @@
+--daemonize=1
+--ca_path=/var/lib/nova/CA
+--keys_path=/var/lib/nova/keys
+--fake_users=1
+--datastore_path=/var/lib/nova/keeper
diff --git a/debian/nova-api.init b/debian/nova-api.init
index 925c92c5e..597fbef95 100644
--- a/debian/nova-api.init
+++ b/debian/nova-api.init
@@ -13,10 +13,10 @@
 set -e
 
 DAEMON=/usr/bin/nova-api
-DAEMON_ARGS="--flagfile=/etc/nova.conf"
+DAEMON_ARGS="--flagfile=/etc/nova/nova-api.conf"
 PIDFILE=/var/run/nova-api.pid
 
-ENABLED=false
+ENABLED=true
 
 if test -f /etc/default/nova-api; then
   . /etc/default/nova-api
diff --git a/debian/nova-api.install b/debian/nova-api.install
index 757235b11..02dbda02d 100644
--- a/debian/nova-api.install
+++ b/debian/nova-api.install
@@ -1 +1,2 @@
 bin/nova-api usr/bin
+debian/nova-api.conf etc/nova
diff --git a/debian/nova-common.dirs b/debian/nova-common.dirs
new file mode 100644
index 000000000..b58fe8b7f
--- /dev/null
+++ b/debian/nova-common.dirs
@@ -0,0 +1,11 @@
+etc/nova
+var/lib/nova/buckets
+var/lib/nova/CA
+var/lib/nova/CA/INTER
+var/lib/nova/CA/newcerts
+var/lib/nova/CA/private
+var/lib/nova/CA/reqs
+var/lib/nova/images
+var/lib/nova/instances
+var/lib/nova/keys
+var/lib/nova/networks
diff --git a/debian/nova-common.install b/debian/nova-common.install
index ab7455314..92b5d3d41 100644
--- a/debian/nova-common.install
+++ b/debian/nova-common.install
@@ -1,5 +1,9 @@
 bin/nova-manage usr/bin
-nova/auth/novarc.template             usr/lib/pymodules/python2.6/nova/auth
-nova/cloudpipe/client.ovpn.template   usr/lib/pymodules/python2.6/nova/cloudpipe
-nova/compute/libvirt.xml.template     usr/lib/pymodules/python2.6/nova/compute
+debian/nova-manage.conf etc/nova
+nova/auth/novarc.template             usr/share/nova
+nova/cloudpipe/client.ovpn.template   usr/share/nova
+nova/compute/libvirt.xml.template     usr/share/nova
 usr/lib/python*/*-packages/nova/*
+CA/openssl.cnf.tmpl                   var/lib/nova/CA
+CA/geninter.sh                        var/lib/nova/CA
+CA/genrootca.sh                       var/lib/nova/CA
diff --git a/debian/nova-compute.conf b/debian/nova-compute.conf
new file mode 100644
index 000000000..5cd3f31ff
--- /dev/null
+++ b/debian/nova-compute.conf
@@ -0,0 +1,9 @@
+--ca_path=/var/lib/nova/CA
+--keys_path=/var/lib/nova/keys
+--datastore_path=/var/lib/nova/keeper
+--instances_path=/var/lib/nova/instances
+--networks_path=/var/lib/nova/networks
+--libvirt_xml_template=/usr/share/nova/libvirt.xml.template
+--vpn_client_template=/usr/share/nova/client.ovpn.template
+--credentials_template=/usr/share/nova/novarc.template
+--fake_users=1
diff --git a/debian/nova-compute.init b/debian/nova-compute.init
index 89d0e5fce..d0f093a7a 100644
--- a/debian/nova-compute.init
+++ b/debian/nova-compute.init
@@ -13,10 +13,10 @@
 set -e
 
 DAEMON=/usr/bin/nova-compute
-DAEMON_ARGS="--flagfile=/etc/nova.conf"
+DAEMON_ARGS="--flagfile=/etc/nova/nova-compute.conf"
 PIDFILE=/var/run/nova-compute.pid
 
-ENABLED=false
+ENABLED=true
 
 if test -f /etc/default/nova-compute; then
   . /etc/default/nova-compute
diff --git a/debian/nova-compute.install b/debian/nova-compute.install
index 6387cef07..5f9df46a8 100644
--- a/debian/nova-compute.install
+++ b/debian/nova-compute.install
@@ -1 +1,2 @@
 bin/nova-compute usr/bin
+debian/nova-compute.conf etc/nova
diff --git a/debian/nova-manage.conf b/debian/nova-manage.conf
new file mode 100644
index 000000000..5ccda7ecf
--- /dev/null
+++ b/debian/nova-manage.conf
@@ -0,0 +1,4 @@
+--ca_path=/var/lib/nova/CA
+--credentials_template=/usr/share/nova/novarc.template
+--keys_path=/var/lib/nova/keys
+--vpn_client_template=/usr/share/nova/client.ovpn.template
diff --git a/debian/nova-objectstore.conf b/debian/nova-objectstore.conf
new file mode 100644
index 000000000..af3271d3b
--- /dev/null
+++ b/debian/nova-objectstore.conf
@@ -0,0 +1,7 @@
+--daemonize=1
+--ca_path=/var/lib/nova/CA
+--keys_path=/var/lib/nova/keys
+--datastore_path=/var/lib/nova/keeper
+--fake_users=1
+--images_path=/var/lib/nova/images
+--buckets_path=/var/lib/nova/buckets
diff --git a/debian/nova-objectstore.init b/debian/nova-objectstore.init
index be7d32d8e..9676345ad 100644
--- a/debian/nova-objectstore.init
+++ b/debian/nova-objectstore.init
@@ -13,10 +13,10 @@
 set -e
 
 DAEMON=/usr/bin/nova-objectstore
-DAEMON_ARGS="--flagfile=/etc/nova.conf"
+DAEMON_ARGS="--flagfile=/etc/nova/nova-objectstore.conf"
 PIDFILE=/var/run/nova-objectstore.pid
 
-ENABLED=false
+ENABLED=true
 
 if test -f /etc/default/nova-objectstore; then
   . /etc/default/nova-objectstore
diff --git a/debian/nova-objectstore.install b/debian/nova-objectstore.install
index ccc60fccc..3ed93ff37 100644
--- a/debian/nova-objectstore.install
+++ b/debian/nova-objectstore.install
@@ -1 +1,3 @@
 bin/nova-objectstore usr/bin
+debian/nova-objectstore.conf etc/nova
+debian/nova-objectstore.nginx.conf etc/nginx/sites-available
diff --git a/debian/nova-objectstore.links b/debian/nova-objectstore.links
new file mode 100644
index 000000000..38e33948e
--- /dev/null
+++ b/debian/nova-objectstore.links
@@ -0,0 +1 @@
+/etc/nginx/sites-available/nova-objectstore.nginx.conf /etc/nginx/sites-enabled/nova-objectstore.nginx.conf
diff --git a/debian/nova-objectstore.nginx.conf b/debian/nova-objectstore.nginx.conf
new file mode 100644
index 000000000..b63424150
--- /dev/null
+++ b/debian/nova-objectstore.nginx.conf
@@ -0,0 +1,17 @@
+server {
+  listen 3333 default;
+  server_name localhost;
+  client_max_body_size 10m;
+
+  access_log /var/log/nginx/localhost.access.log;
+
+  location ~ /_images/.+ {
+    root /var/lib/nova/images;
+    rewrite ^/_images/(.*)$ /$1 break;
+  }
+
+  location / {
+    proxy_pass http://localhost:3334/;
+  }
+}
+
diff --git a/debian/nova-volume.conf b/debian/nova-volume.conf
new file mode 100644
index 000000000..af3271d3b
--- /dev/null
+++ b/debian/nova-volume.conf
@@ -0,0 +1,7 @@
+--daemonize=1
+--ca_path=/var/lib/nova/CA
+--keys_path=/var/lib/nova/keys
+--datastore_path=/var/lib/nova/keeper
+--fake_users=1
+--images_path=/var/lib/nova/images
+--buckets_path=/var/lib/nova/buckets
diff --git a/debian/nova-volume.init b/debian/nova-volume.init
index 80da3f70c..d5c2dddf8 100644
--- a/debian/nova-volume.init
+++ b/debian/nova-volume.init
@@ -13,10 +13,10 @@
 set -e
 
 DAEMON=/usr/bin/nova-volume
-DAEMON_ARGS="--flagfile=/etc/nova.conf"
+DAEMON_ARGS="--flagfile=/etc/nova/nova-volume.conf"
 PIDFILE=/var/run/nova-volume.pid
 
-ENABLED=false
+ENABLED=true
 
 if test -f /etc/default/nova-volume; then
   . /etc/default/nova-volume
diff --git a/debian/nova-volume.install b/debian/nova-volume.install
index 37b535c03..9a840c78e 100644
--- a/debian/nova-volume.install
+++ b/debian/nova-volume.install
@@ -1 +1,2 @@
 bin/nova-volume usr/bin
+debian/nova-volume.conf etc/nova
diff --git a/nova/compute/disk.py b/nova/compute/disk.py
index bd6a010ee..02768755f 100644
--- a/nova/compute/disk.py
+++ b/nova/compute/disk.py
@@ -126,8 +126,7 @@ def inject_key(key, image, partition=None, execute=None):
                 yield execute('sudo umount %s' % mapped_device)
         finally:
             # remove temporary directory
-            # TODO(termie): scary, is there any thing we can check here?
-            yield execute('rm -rf %s' % tmpdir)
+            yield execute('rmdir %s' % tmpdir)
             if not partition is None:
                 # remove partitions
                 yield execute('sudo kpartx -d %s' % device)
diff --git a/nova/compute/network.py b/nova/compute/network.py
index 911d0344a..94fa6c47a 100644
--- a/nova/compute/network.py
+++ b/nova/compute/network.py
@@ -133,7 +133,7 @@ class BaseNetwork(datastore.RedisModel):
 
     @property
     def hosts(self):
-        return datastore.Redis.instance().hgetall(self._hosts_key)
+        return datastore.Redis.instance().hgetall(self._hosts_key) or {}
 
     def _add_host(self, _user_id, _project_id, host, target):
         datastore.Redis.instance().hset(self._hosts_key, host, target)
@@ -392,7 +392,7 @@ def _rem_vlan(project_id):
 
 def get_assigned_vlans():
     """ Returns a dictionary, with keys of project_id and values of vlan_id """
-    return datastore.Redis.instance().hgetall(VLANS_KEY)
+    return datastore.Redis.instance().hgetall(VLANS_KEY) or {}
 
 def get_vlan_for_project(project_id):
     """
diff --git a/nova/crypto.py b/nova/crypto.py
index 92a1ab76f..fc6ed714f 100644
--- a/nova/crypto.py
+++ b/nova/crypto.py
@@ -23,10 +23,12 @@ Wrappers around standard crypto, including root and intermediate CAs,
 SSH keypairs and x509 certificates.
 """
 
+import base64
 import hashlib
 import logging
 import os
 import shutil
+import struct
 import tempfile
 import time
 import utils
@@ -86,14 +88,17 @@ def generate_key_pair(bits=1024):
 
 
 def ssl_pub_to_ssh_pub(ssl_public_key, name='root', suffix='nova'):
-    """requires lsh-utils"""
-    convert="sed -e'1d' -e'$d' |  pkcs1-conv --public-key-info --base-64 |" \
-    + " sexp-conv |  sed -e'1s/(rsa-pkcs1/(rsa-pkcs1-sha1/' |  sexp-conv -s" \
-    + " transport | lsh-export-key --openssh"
-    (out, err) = utils.execute(convert, ssl_public_key)
-    if err:
-        raise exception.Error("Failed to generate key: %s" % err)
-    return '%s %s@%s\n' %(out.strip(), name, suffix)
+    rsa_key = M2Crypto.RSA.load_pub_key_bio(M2Crypto.BIO.MemoryBuffer(ssl_public_key))
+    e, n = rsa_key.pub()
+
+    key_type = 'ssh-rsa'
+
+    key_data = struct.pack('>I', len(key_type))
+    key_data += key_type
+    key_data += '%s%s' % (e,n)
+
+    b64_blob = base64.b64encode(key_data)
+    return '%s %s %s@%s\n' %(key_type, b64_blob, name, suffix)
 
 
 def generate_x509_cert(subject, bits=1024):
diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py
index 19b6db91f..4a3849cb0 100644
--- a/nova/endpoint/cloud.py
+++ b/nova/endpoint/cloud.py
@@ -523,6 +523,10 @@ class CloudController(object):
             inst = self.instdir.new()
             # TODO(ja): add ari, aki
             inst['image_id'] = kwargs['image_id']
+            if 'kernel_id' in kwargs:
+                inst['kernel_id'] = kwargs['kernel_id']
+            if 'ramdisk_id' in kwargs:
+                inst['ramdisk_id'] = kwargs['ramdisk_id']
             inst['user_data'] = kwargs.get('user_data', '')
             inst['instance_type'] = kwargs.get('instance_type', 'm1.small')
             inst['reservation_id'] = reservation_id