merged trunk

13 files changed, 91 insertions, 70 deletions

diff --git a/bin/nova-objectstore b/bin/nova-objectstore
index 9385fd299..c0fa815c0 100755
--- a/bin/nova-objectstore
+++ b/bin/nova-objectstore
@@ -21,8 +21,6 @@
   Twisted daemon for nova objectstore. Supports S3 API.
 """
 
-import logging
-
 from nova import flags
 from nova import utils
 from nova import twistd
@@ -33,9 +31,6 @@ FLAGS = flags.FLAGS
 
 
 def main():
-    # FIXME: if this log statement isn't here, no logging
-    # appears from other files and app won't start daemonized
-    logging.debug('Started HTTP server on %s' % (FLAGS.s3_port))
     app = handler.get_application()
     print app
     return app
diff --git a/nova/adminclient.py b/nova/adminclient.py
index fceeac274..25d5e71cb 100644
--- a/nova/adminclient.py
+++ b/nova/adminclient.py
@@ -292,9 +292,13 @@ class NovaAdminClient(object):
                   'Operation': operation}
         return self.apiconn.get_status('ModifyProjectMember', params)
 
-    def get_zip(self, username):
-        """ returns the content of a zip file containing novarc and access credentials. """
-        return self.apiconn.get_object('GenerateX509ForUser', {'Name': username}, UserInfo).file
+    def get_zip(self, user, project):
+        """
+        Returns the content of a zip file containing novarc and access credentials.
+        """
+        params = {'Name': user, 'Project': project}
+        zip = self.apiconn.get_object('GenerateX509ForUser', params, UserInfo)
+        return zip.file
 
     def get_hosts(self):
         return self.apiconn.get_list('DescribeHosts', {}, [('item', HostInfo)])
diff --git a/nova/auth/manager.py b/nova/auth/manager.py
index cf920d607..d44ed52b2 100644
--- a/nova/auth/manager.py
+++ b/nova/auth/manager.py
@@ -24,7 +24,6 @@ import logging
 import os
 import shutil
 import string
-import sys
 import tempfile
 import uuid
 import zipfile
@@ -239,8 +238,7 @@ class AuthManager(object):
     def __new__(cls, *args, **kwargs):
         """Returns the AuthManager singleton"""
         if not cls._instance:
-            cls._instance = super(AuthManager, cls).__new__(
-                    cls, *args, **kwargs)
+            cls._instance = super(AuthManager, cls).__new__(cls)
         return cls._instance
 
     def __init__(self, driver=None, *args, **kwargs):
@@ -333,6 +331,12 @@ class AuthManager(object):
                 raise exception.NotAuthorized('Signature does not match')
         return (user, project)
 
+    def get_access_key(self, user, project):
+        """Get an access key that includes user and project"""
+        if not isinstance(user, User):
+            user = self.get_user(user)
+        return "%s:%s" % (user.access, Project.safe_id(project))
+
     def is_superuser(self, user):
         """Checks for superuser status, allowing user to bypass rbac
 
diff --git a/nova/auth/signer.py b/nova/auth/signer.py
index 7d7471575..634f22f0d 100644
--- a/nova/auth/signer.py
+++ b/nova/auth/signer.py
@@ -34,7 +34,7 @@
 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
 # OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
 # ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
-# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 
+# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 # IN THE SOFTWARE.
@@ -48,7 +48,8 @@ import hashlib
 import hmac
 import logging
 import urllib
-import boto.utils
+import boto       # NOTE(vish): for new boto
+import boto.utils # NOTE(vish): for old boto
 
 from nova.exception import Error
 
diff --git a/nova/compute/libvirt.xml.template b/nova/compute/libvirt.xml.template
index a763e8a4d..307f9d03a 100644
--- a/nova/compute/libvirt.xml.template
+++ b/nova/compute/libvirt.xml.template
@@ -1,4 +1,4 @@
-<domain type='kvm'>
+<domain type='%(type)s'>
     <name>%(name)s</name>
     <os>
     <type>hvm</type>
@@ -12,7 +12,6 @@
     <memory>%(memory_kb)s</memory>
     <vcpu>%(vcpus)s</vcpu>
     <devices>
-        <emulator>/usr/bin/kvm</emulator>
         <disk type='file'>
             <source file='%(basepath)s/disk'/>
             <target dev='vda' bus='virtio'/>
diff --git a/nova/endpoint/cloud.py b/nova/endpoint/cloud.py
index 00dabba28..23ad85cd3 100644
--- a/nova/endpoint/cloud.py
+++ b/nova/endpoint/cloud.py
@@ -294,17 +294,16 @@ class CloudController(object):
         return v
 
     @rbac.allow('projectmanager', 'sysadmin')
+    @defer.inlineCallbacks
     def create_volume(self, context, size, **kwargs):
         # TODO(vish): refactor this to create the volume object here and tell service to create it
-        res = rpc.call(FLAGS.volume_topic, {"method": "create_volume",
+        result = yield rpc.call(FLAGS.volume_topic, {"method": "create_volume",
                                  "args" : {"size": size,
                                            "user_id": context.user.id,
                                            "project_id": context.project.id}})
-        def _format_result(result):
-            volume = self._get_volume(context, result['result'])
-            return {'volumeSet': [self.format_volume(context, volume)]}
-        res.addCallback(_format_result)
-        return res
+        # NOTE(vish): rpc returned value is in the result key in the dictionary
+        volume = self._get_volume(context, result['result'])
+        defer.returnValue({'volumeSet': [self.format_volume(context, volume)]})
 
     def _get_address(self, context, public_ip):
         # FIXME(vish) this should move into network.py
diff --git a/nova/endpoint/images.py b/nova/endpoint/images.py
index 12876da30..fe7cb5d11 100644
--- a/nova/endpoint/images.py
+++ b/nova/endpoint/images.py
@@ -21,14 +21,13 @@ Proxy AMI-related calls from the cloud controller, to the running
 objectstore daemon.
 """
 
-import boto
-import boto.s3
+import boto.s3.connection
 import json
-import random
 import urllib
 
 from nova import flags
 from nova import utils
+from nova.auth import manager
 
 
 FLAGS = flags.FLAGS
@@ -77,13 +76,16 @@ def deregister(context, image_id):
             query_args=qs({'image_id': image_id}))
 
 def conn(context):
-    return boto.s3.connection.S3Connection (
-        aws_access_key_id=str('%s:%s' % (context.user.access, context.project.name)),
-        aws_secret_access_key=str(context.user.secret),
-        is_secure=False,
-        calling_format=boto.s3.connection.OrdinaryCallingFormat(),
-        port=FLAGS.s3_port,
-        host=FLAGS.s3_host)
+    access = manager.AuthManager().get_access_key(context.user,
+                                                  context.project)
+    secret = str(context.user.secret)
+    calling = boto.s3.connection.OrdinaryCallingFormat()
+    return boto.s3.connection.S3Connection(aws_access_key_id=access,
+                                           aws_secret_access_key=secret,
+                                           is_secure=False,
+                                           calling_format=calling,
+                                           port=FLAGS.s3_port,
+                                           host=FLAGS.s3_host)
 
 
 def qs(params):
diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py
index b536ac383..0f4f0e34d 100644
--- a/nova/tests/volume_unittest.py
+++ b/nova/tests/volume_unittest.py
@@ -42,15 +42,14 @@ class VolumeTestCase(test.TrialTestCase):
         vol_size = '0'
         user_id = 'fake'
         project_id = 'fake'
-        volume_id = self.volume.create_volume(vol_size, user_id, project_id)
+        volume_id = yield self.volume.create_volume(vol_size, user_id, project_id)
         # TODO(termie): get_volume returns differently than create_volume
         self.assertEqual(volume_id,
                          volume_service.get_volume(volume_id)['volume_id'])
 
         rv = self.volume.delete_volume(volume_id)
-        self.assertRaises(exception.Error,
-                          volume_service.get_volume,
-                          volume_id)
+        self.assertFailure(volume_service.get_volume(volume_id),
+                           exception.Error)
 
     def test_too_big_volume(self):
         vol_size = '1001'
@@ -68,13 +67,14 @@ class VolumeTestCase(test.TrialTestCase):
         total_slots = FLAGS.slots_per_shelf * num_shelves
         vols = []
         for i in xrange(total_slots):
-            vid = self.volume.create_volume(vol_size, user_id, project_id)
+            vid = yield self.volume.create_volume(vol_size, user_id, project_id)
             vols.append(vid)
-        self.assertRaises(volume_service.NoMoreVolumes,
-                          self.volume.create_volume,
-                          vol_size, user_id, project_id)
+        self.assertFailure(self.volume.create_volume(vol_size,
+                                                     user_id,
+                                                     project_id),
+                           volume_service.NoMoreVolumes)
         for id in vols:
-            self.volume.delete_volume(id)
+            yield self.volume.delete_volume(id)
 
     def test_run_attach_detach_volume(self):
         # Create one volume and one compute to test with
@@ -83,7 +83,7 @@ class VolumeTestCase(test.TrialTestCase):
         user_id = "fake"
         project_id = 'fake'
         mountpoint = "/dev/sdf"
-        volume_id = self.volume.create_volume(vol_size, user_id, project_id)
+        volume_id = yield self.volume.create_volume(vol_size, user_id, project_id)
 
         volume_obj = volume_service.get_volume(volume_id)
         volume_obj.start_attach(instance_id, mountpoint)
diff --git a/nova/twistd.py b/nova/twistd.py
index ecb6e2892..c83276daa 100644
--- a/nova/twistd.py
+++ b/nova/twistd.py
@@ -214,6 +214,9 @@ def serve(filename):
         FLAGS.pidfile = '%s.pid' % name
     elif FLAGS.pidfile.endswith('twistd.pid'):
         FLAGS.pidfile = FLAGS.pidfile.replace('twistd.pid', '%s.pid' % name)
+    # NOTE(vish): if we're running nodaemon, redirect the log to stdout
+    if FLAGS.nodaemon and not FLAGS.logfile:
+        FLAGS.logfile = "-"
     if not FLAGS.logfile:
         FLAGS.logfile = '%s.log' % name
     elif FLAGS.logfile.endswith('twistd.log'):
diff --git a/nova/utils.py b/nova/utils.py
index 0016b656e..0b23de7cd 100644
--- a/nova/utils.py
+++ b/nova/utils.py
@@ -41,7 +41,7 @@ def import_class(import_str):
     try:
         __import__(mod_str)
         return getattr(sys.modules[mod_str], class_str)
-    except (ImportError, AttributeError):
+    except (ImportError, ValueError, AttributeError):
         raise exception.NotFound('Class %s cannot be found' % class_str)
 
 def fetchfile(url, target):
diff --git a/nova/virt/images.py b/nova/virt/images.py
index 92210e242..872eb6d6a 100644
--- a/nova/virt/images.py
+++ b/nova/virt/images.py
@@ -27,6 +27,7 @@ import time
 from nova import flags
 from nova import process
 from nova.auth import signer
+from nova.auth import manager
 
 FLAGS = flags.FLAGS
 
@@ -34,14 +35,14 @@ flags.DEFINE_bool('use_s3', True,
                   'whether to get images from s3 or use local copy')
 
 
-def fetch(image, path, user):
+def fetch(image, path, user, project):
     if FLAGS.use_s3:
         f = _fetch_s3_image
     else:
         f = _fetch_local_image
-    return f(image, path, user)
+    return f(image, path, user, project)
 
-def _fetch_s3_image(image, path, user):
+def _fetch_s3_image(image, path, user, project):
     url = _image_url('%s/image' % image)
 
     # This should probably move somewhere else, like e.g. a download_as
@@ -51,8 +52,11 @@ def _fetch_s3_image(image, path, user):
     headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime())
 
     uri = '/' + url.partition('/')[2]
-    auth = signer.Signer(user.secret.encode()).s3_authorization(headers, 'GET', uri)
-    headers['Authorization'] = 'AWS %s:%s' % (user.access, auth)
+    access = manager.AuthManager().get_access_key(user, project)
+    signature = signer.Signer(user.secret.encode()).s3_authorization(headers,
+                                                                     'GET',
+                                                                     uri)
+    headers['Authorization'] = 'AWS %s:%s' % (access, signature)
 
     cmd = ['/usr/bin/curl', '--silent', url]
     for (k,v) in headers.iteritems():
diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py
index 7c3f7a6e1..551ba6e54 100644
--- a/nova/virt/libvirt_conn.py
+++ b/nova/virt/libvirt_conn.py
@@ -25,7 +25,6 @@ import json
 import logging
 import os.path
 import shutil
-import sys
 
 from twisted.internet import defer
 from twisted.internet import task
@@ -51,6 +50,10 @@ flags.DEFINE_string('injected_network_template',
                     utils.abspath('compute/interfaces.template'),
                     'Template file for injected network')
 
+flags.DEFINE_string('libvirt_type',
+                    'kvm',
+                    'Libvirt domain type (kvm, qemu, etc)')
+
 def get_connection(read_only):
     # These are loaded late so that there's no need to install these
     # libraries when not using libvirt.
@@ -190,12 +193,13 @@ class LibvirtConnection(object):
         f.close()
 
         user = manager.AuthManager().get_user(data['user_id'])
+        project = manager.AuthManager().get_project(data['project_id'])
         if not os.path.exists(basepath('disk')):
-           yield images.fetch(data['image_id'], basepath('disk-raw'), user)
+           yield images.fetch(data['image_id'], basepath('disk-raw'), user, project)
         if not os.path.exists(basepath('kernel')):
-           yield images.fetch(data['kernel_id'], basepath('kernel'), user)
+           yield images.fetch(data['kernel_id'], basepath('kernel'), user, project)
         if not os.path.exists(basepath('ramdisk')):
-           yield images.fetch(data['ramdisk_id'], basepath('ramdisk'), user)
+           yield images.fetch(data['ramdisk_id'], basepath('ramdisk'), user, project)
 
         execute = lambda cmd, input=None: \
                   process.simple_execute(cmd=cmd,
@@ -238,6 +242,7 @@ class LibvirtConnection(object):
 
         # TODO(termie): lazy lazy hack because xml is annoying
         xml_info['nova'] = json.dumps(instance.datamodel.copy())
+        xml_info['type'] = FLAGS.libvirt_type
         libvirt_xml = libvirt_xml % xml_info
         logging.debug("Finished the toXML method")
 
diff --git a/nova/volume/service.py b/nova/volume/service.py
index 87a47f40a..e12f675a7 100644
--- a/nova/volume/service.py
+++ b/nova/volume/service.py
@@ -103,6 +103,7 @@ class VolumeService(service.Service):
             except Exception, err:
                 pass
 
+    @defer.inlineCallbacks
     @validate.rangetest(size=(0, 1000))
     def create_volume(self, size, user_id, project_id):
         """
@@ -111,11 +112,12 @@ class VolumeService(service.Service):
         Volume at this point has size, owner, and zone.
         """
         logging.debug("Creating volume of size: %s" % (size))
-        vol = self.volume_class.create(size, user_id, project_id)
+        vol = yield self.volume_class.create(size, user_id, project_id)
         datastore.Redis.instance().sadd('volumes', vol['volume_id'])
         datastore.Redis.instance().sadd('volumes:%s' % (FLAGS.storage_name), vol['volume_id'])
-        self._restart_exports()
-        return vol['volume_id']
+        logging.debug("restarting exports")
+        yield self._restart_exports()
+        defer.returnValue(vol['volume_id'])
 
     def by_node(self, node_id):
         """ returns a list of volumes for a node """
@@ -128,6 +130,7 @@ class VolumeService(service.Service):
         for volume_id in datastore.Redis.instance().smembers('volumes'):
             yield self.volume_class(volume_id=volume_id)
 
+    @defer.inlineCallbacks
     def delete_volume(self, volume_id):
         logging.debug("Deleting volume with id of: %s" % (volume_id))
         vol = get_volume(volume_id)
@@ -135,19 +138,18 @@ class VolumeService(service.Service):
             raise exception.Error("Volume is still attached")
         if vol['node_name'] != FLAGS.storage_name:
             raise exception.Error("Volume is not local to this node")
-        vol.destroy()
+        yield vol.destroy()
         datastore.Redis.instance().srem('volumes', vol['volume_id'])
         datastore.Redis.instance().srem('volumes:%s' % (FLAGS.storage_name), vol['volume_id'])
-        return True
+        defer.returnValue(True)
 
     @defer.inlineCallbacks
     def _restart_exports(self):
         if FLAGS.fake_storage:
             return
-        yield process.simple_execute(
-                "sudo vblade-persist auto all")
-        yield process.simple_execute(
-                "sudo vblade-persist start all")
+        yield process.simple_execute("sudo vblade-persist auto all")
+        # NOTE(vish): this command sometimes sends output to stderr for warnings
+        yield process.simple_execute("sudo vblade-persist start all", error_ok=1)
 
     @defer.inlineCallbacks
     def _init_volume_group(self):
@@ -173,6 +175,7 @@ class Volume(datastore.BasicModel):
         return {"volume_id": self.volume_id}
 
     @classmethod
+    @defer.inlineCallbacks
     def create(cls, size, user_id, project_id):
         volume_id = utils.generate_uid('vol')
         vol = cls(volume_id)
@@ -188,13 +191,12 @@ class Volume(datastore.BasicModel):
         vol['attach_status'] = "detached"  # attaching | attached | detaching | detached
         vol['delete_on_termination'] = 'False'
         vol.save()
-        vol.create_lv()
-        vol._setup_export()
+        yield vol._create_lv()
+        yield vol._setup_export()
         # TODO(joshua) - We need to trigger a fanout message for aoe-discover on all the nodes
-        # TODO(joshua
         vol['status'] = "available"
         vol.save()
-        return vol
+        defer.returnValue(vol)
 
     def start_attach(self, instance_id, mountpoint):
         """ """
@@ -223,16 +225,18 @@ class Volume(datastore.BasicModel):
         self['attach_status'] = "detached"
         self.save()
 
+    @defer.inlineCallbacks
     def destroy(self):
         try:
-            self._remove_export()
-        except:
+            yield self._remove_export()
+        except Exception as ex:
+            logging.debug("Ingnoring failure to remove export %s" % ex)
             pass
-        self._delete_lv()
+        yield self._delete_lv()
         super(Volume, self).destroy()
 
     @defer.inlineCallbacks
-    def create_lv(self):
+    def _create_lv(self):
         if str(self['size']) == '0':
             sizestr = '100M'
         else:
@@ -248,13 +252,14 @@ class Volume(datastore.BasicModel):
                 "sudo lvremove -f %s/%s" % (FLAGS.volume_group,
                                             self['volume_id']))
 
+    @defer.inlineCallbacks
     def _setup_export(self):
         (shelf_id, blade_id) = get_next_aoe_numbers()
         self['aoe_device'] = "e%s.%s" % (shelf_id, blade_id)
         self['shelf_id'] = shelf_id
         self['blade_id'] = blade_id
         self.save()
-        self._exec_export()
+        yield self._exec_export()
 
     @defer.inlineCallbacks
     def _exec_export(self):
@@ -277,7 +282,7 @@ class Volume(datastore.BasicModel):
 
 
 class FakeVolume(Volume):
-    def create_lv(self):
+    def _create_lv(self):
         pass
 
     def _exec_export(self):