Merge from trunk

4 files changed, 536 insertions, 12 deletions

diff --git a/bin/instance-usage-audit b/bin/instance-usage-audit
index a06c6b1b3..7ce5732e7 100755
--- a/bin/instance-usage-audit
+++ b/bin/instance-usage-audit
@@ -102,9 +102,8 @@ if __name__ == '__main__':
     logging.setup()
     begin, end = time_period(FLAGS.instance_usage_audit_period)
     print "Creating usages for %s until %s" % (str(begin), str(end))
-    instances = db.instance_get_active_by_window(context.get_admin_context(),
-                                                 begin,
-                                                 end)
+    ctxt = context.get_admin_context()
+    instances = db.instance_get_active_by_window_joined(ctxt, begin, end)
     print "%s instances" % len(instances)
     for instance_ref in instances:
         usage_info = utils.usage_from_instance(instance_ref,
diff --git a/bin/nova-ajax-console-proxy b/bin/nova-ajax-console-proxy
index 0a789b4b9..23fb42fb5 100755
--- a/bin/nova-ajax-console-proxy
+++ b/bin/nova-ajax-console-proxy
@@ -113,11 +113,10 @@ class AjaxConsoleProxy(object):
                 AjaxConsoleProxy.tokens[kwargs['token']] =  \
                     {'args': kwargs, 'last_activity': time.time()}
 
-        conn = rpc.create_connection(new=True)
-        consumer = rpc.create_consumer(
-                       conn,
-                       FLAGS.ajax_console_proxy_topic,
-                       TopicProxy)
+        self.conn = rpc.create_connection(new=True)
+        self.conn.create_consumer(
+                FLAGS.ajax_console_proxy_topic,
+                TopicProxy)
 
         def delete_expired_tokens():
             now = time.time()
@@ -129,7 +128,7 @@ class AjaxConsoleProxy(object):
             for k in to_delete:
                 del AjaxConsoleProxy.tokens[k]
 
-        utils.LoopingCall(consumer.fetch, enable_callbacks=True).start(0.1)
+        self.conn.consume_in_thread()
         utils.LoopingCall(delete_expired_tokens).start(1)
 
 if __name__ == '__main__':
@@ -142,3 +141,4 @@ if __name__ == '__main__':
     server = wsgi.Server("AJAX Console Proxy", acp, port=acp_port)
     service.serve(server)
     service.wait()
+    self.conn.close()
diff --git a/bin/nova-manage b/bin/nova-manage
index 890cde0b8..c3b2c71ce 100755
--- a/bin/nova-manage
+++ b/bin/nova-manage
@@ -53,6 +53,7 @@
   CLI interface for nova management.
 """
 
+import ast
 import gettext
 import glob
 import json
@@ -85,11 +86,13 @@ from nova import quota
 from nova import rpc
 from nova import utils
 from nova import version
+from nova import vsa
 from nova.api.ec2 import ec2utils
 from nova.auth import manager
 from nova.cloudpipe import pipelib
 from nova.compute import instance_types
 from nova.db import migration
+from nova.volume import volume_types
 
 FLAGS = flags.FLAGS
 flags.DECLARE('fixed_range', 'nova.network.manager')
@@ -163,7 +166,7 @@ class VpnCommands(object):
                 print address,
                 print vpn['host'],
                 print ec2utils.id_to_ec2_id(vpn['id']),
-                print vpn['state_description'],
+                print vpn['vm_state'],
                 print state
             else:
                 print None
@@ -866,7 +869,7 @@ class VmCommands(object):
                 instance['hostname'],
                 instance['host'],
                 instance['instance_type'].name,
-                instance['state_description'],
+                instance['vm_state'],
                 instance['launched_at'],
                 instance['image_ref'],
                 instance['kernel_id'],
@@ -1097,6 +1100,477 @@ class VersionCommands(object):
         self.list()
 
 
+class VsaCommands(object):
+    """Methods for dealing with VSAs"""
+
+    def __init__(self, *args, **kwargs):
+        self.manager = manager.AuthManager()
+        self.vsa_api = vsa.API()
+        self.context = context.get_admin_context()
+
+        self._format_str_vsa = "%(id)-5s %(vsa_id)-15s %(name)-25s "\
+                    "%(type)-10s %(vcs)-6s %(drives)-9s %(stat)-10s "\
+                    "%(az)-10s %(time)-10s"
+        self._format_str_volume = "\t%(id)-4s %(name)-15s %(size)-5s "\
+                    "%(stat)-10s %(att)-20s %(time)s"
+        self._format_str_drive = "\t%(id)-4s %(name)-15s %(size)-5s "\
+                    "%(stat)-10s %(host)-20s %(type)-4s %(tname)-10s %(time)s"
+        self._format_str_instance = "\t%(id)-4s %(name)-10s %(dname)-20s "\
+                    "%(image)-12s %(type)-10s %(fl_ip)-15s %(fx_ip)-15s "\
+                    "%(stat)-10s %(host)-15s %(time)s"
+
+    def _print_vsa_header(self):
+        print self._format_str_vsa %\
+               dict(id=_('ID'),
+                    vsa_id=_('vsa_id'),
+                    name=_('displayName'),
+                    type=_('vc_type'),
+                    vcs=_('vc_cnt'),
+                    drives=_('drive_cnt'),
+                    stat=_('status'),
+                    az=_('AZ'),
+                    time=_('createTime'))
+
+    def _print_vsa(self, vsa):
+        print self._format_str_vsa %\
+                dict(id=vsa['id'],
+                     vsa_id=vsa['name'],
+                     name=vsa['display_name'],
+                     type=vsa['vsa_instance_type'].get('name', None),
+                     vcs=vsa['vc_count'],
+                     drives=vsa['vol_count'],
+                     stat=vsa['status'],
+                     az=vsa['availability_zone'],
+                     time=str(vsa['created_at']))
+
+    def _print_volume_header(self):
+        print _('      === Volumes ===')
+        print self._format_str_volume %\
+                dict(id=_('ID'),
+                     name=_('name'),
+                     size=_('size'),
+                     stat=_('status'),
+                     att=_('attachment'),
+                     time=_('createTime'))
+
+    def _print_volume(self, vol):
+        print self._format_str_volume %\
+                dict(id=vol['id'],
+                     name=vol['display_name'] or vol['name'],
+                     size=vol['size'],
+                     stat=vol['status'],
+                     att=vol['attach_status'],
+                     time=str(vol['created_at']))
+
+    def _print_drive_header(self):
+        print _('      === Drives ===')
+        print self._format_str_drive %\
+                dict(id=_('ID'),
+                    name=_('name'),
+                    size=_('size'),
+                    stat=_('status'),
+                    host=_('host'),
+                    type=_('type'),
+                    tname=_('typeName'),
+                    time=_('createTime'))
+
+    def _print_drive(self, drive):
+        if drive['volume_type_id'] is not None and drive.get('volume_type'):
+            drive_type_name = drive['volume_type'].get('name')
+        else:
+            drive_type_name = ''
+
+        print self._format_str_drive %\
+                dict(id=drive['id'],
+                    name=drive['display_name'],
+                    size=drive['size'],
+                    stat=drive['status'],
+                    host=drive['host'],
+                    type=drive['volume_type_id'],
+                    tname=drive_type_name,
+                    time=str(drive['created_at']))
+
+    def _print_instance_header(self):
+        print _('      === Instances ===')
+        print self._format_str_instance %\
+                dict(id=_('ID'),
+                    name=_('name'),
+                    dname=_('disp_name'),
+                    image=_('image'),
+                    type=_('type'),
+                    fl_ip=_('floating_IP'),
+                    fx_ip=_('fixed_IP'),
+                    stat=_('status'),
+                    host=_('host'),
+                    time=_('createTime'))
+
+    def _print_instance(self, vc):
+
+        fixed_addr = None
+        floating_addr = None
+        if vc['fixed_ips']:
+            fixed = vc['fixed_ips'][0]
+            fixed_addr = fixed['address']
+            if fixed['floating_ips']:
+                floating_addr = fixed['floating_ips'][0]['address']
+        floating_addr = floating_addr or fixed_addr
+
+        print self._format_str_instance %\
+                dict(id=vc['id'],
+                    name=ec2utils.id_to_ec2_id(vc['id']),
+                    dname=vc['display_name'],
+                    image=('ami-%08x' % int(vc['image_ref'])),
+                    type=vc['instance_type']['name'],
+                    fl_ip=floating_addr,
+                    fx_ip=fixed_addr,
+                    stat=vc['vm_state'],
+                    host=vc['host'],
+                    time=str(vc['created_at']))
+
+    def _list(self, context, vsas, print_drives=False,
+                    print_volumes=False, print_instances=False):
+        if vsas:
+            self._print_vsa_header()
+
+        for vsa in vsas:
+            self._print_vsa(vsa)
+            vsa_id = vsa.get('id')
+
+            if print_instances:
+                instances = self.vsa_api.get_all_vsa_instances(context, vsa_id)
+                if instances:
+                    print
+                    self._print_instance_header()
+                    for instance in instances:
+                        self._print_instance(instance)
+                    print
+
+            if print_drives:
+                drives = self.vsa_api.get_all_vsa_drives(context, vsa_id)
+                if drives:
+                    self._print_drive_header()
+                    for drive in drives:
+                        self._print_drive(drive)
+                    print
+
+            if print_volumes:
+                volumes = self.vsa_api.get_all_vsa_volumes(context, vsa_id)
+                if volumes:
+                    self._print_volume_header()
+                    for volume in volumes:
+                        self._print_volume(volume)
+                    print
+
+    @args('--storage', dest='storage',
+        metavar="[{'drive_name': 'type', 'num_drives': N, 'size': M},..]",
+        help='Initial storage allocation for VSA')
+    @args('--name', dest='name', metavar="<name>", help='VSA name')
+    @args('--description', dest='description', metavar="<description>",
+        help='VSA description')
+    @args('--vc', dest='vc_count', metavar="<number>", help='Number of VCs')
+    @args('--instance_type', dest='instance_type_name', metavar="<name>",
+        help='Instance type name')
+    @args('--image', dest='image_name', metavar="<name>", help='Image name')
+    @args('--shared', dest='shared', action="store_true", default=False,
+        help='Use shared drives')
+    @args('--az', dest='az', metavar="<zone:host>", help='Availability zone')
+    @args('--user', dest="user_id", metavar='<User name>',
+            help='User name')
+    @args('--project', dest="project_id", metavar='<Project name>',
+            help='Project name')
+    def create(self, storage='[]', name=None, description=None, vc_count=1,
+                     instance_type_name=None, image_name=None, shared=None,
+                     az=None, user_id=None, project_id=None):
+        """Create a VSA."""
+
+        if project_id is None:
+            try:
+                project_id = os.getenv("EC2_ACCESS_KEY").split(':')[1]
+            except Exception as exc:
+                print _("Failed to retrieve project id: %(exc)s") % exc
+                raise
+
+        if user_id is None:
+            try:
+                project = self.manager.get_project(project_id)
+                user_id = project.project_manager_id
+            except Exception as exc:
+                print _("Failed to retrieve user info: %(exc)s") % exc
+                raise
+
+        is_admin = self.manager.is_admin(user_id)
+        ctxt = context.RequestContext(user_id, project_id, is_admin)
+        if not is_admin and \
+           not self.manager.is_project_member(user_id, project_id):
+            msg = _("%(user_id)s must be an admin or a "
+                    "member of %(project_id)s")
+            LOG.warn(msg % locals())
+            raise ValueError(msg % locals())
+
+        # Sanity check for storage string
+        storage_list = []
+        if storage is not None:
+            try:
+                storage_list = ast.literal_eval(storage)
+            except:
+                print _("Invalid string format %s") % storage
+                raise
+
+            for node in storage_list:
+                if ('drive_name' not in node) or ('num_drives' not in node):
+                    print (_("Invalid string format for element %s. " \
+                            "Expecting keys 'drive_name' & 'num_drives'"),
+                            str(node))
+                    raise KeyError
+
+        if instance_type_name == '':
+            instance_type_name = None
+        instance_type = instance_types.get_instance_type_by_name(
+                                instance_type_name)
+
+        if image_name == '':
+            image_name = None
+
+        if shared in [None, False, "--full_drives"]:
+            shared = False
+        elif shared in [True, "--shared"]:
+            shared = True
+        else:
+            raise ValueError(_('Shared parameter should be set either to "\
+                "--shared or --full_drives'))
+
+        values = {
+            'display_name': name,
+            'display_description': description,
+            'vc_count': int(vc_count),
+            'instance_type': instance_type,
+            'image_name': image_name,
+            'availability_zone': az,
+            'storage': storage_list,
+            'shared': shared,
+            }
+
+        result = self.vsa_api.create(ctxt, **values)
+        self._list(ctxt, [result])
+
+    @args('--id', dest='vsa_id', metavar="<vsa_id>", help='VSA ID')
+    @args('--name', dest='name', metavar="<name>", help='VSA name')
+    @args('--description', dest='description', metavar="<description>",
+        help='VSA description')
+    @args('--vc', dest='vc_count', metavar="<number>", help='Number of VCs')
+    def update(self, vsa_id, name=None, description=None, vc_count=None):
+        """Updates name/description of vsa and number of VCs."""
+
+        values = {}
+        if name is not None:
+            values['display_name'] = name
+        if description is not None:
+            values['display_description'] = description
+        if vc_count is not None:
+            values['vc_count'] = int(vc_count)
+
+        vsa_id = ec2utils.ec2_id_to_id(vsa_id)
+        result = self.vsa_api.update(self.context, vsa_id=vsa_id, **values)
+        self._list(self.context, [result])
+
+    @args('--id', dest='vsa_id', metavar="<vsa_id>", help='VSA ID')
+    def delete(self, vsa_id):
+        """Delete a VSA."""
+        vsa_id = ec2utils.ec2_id_to_id(vsa_id)
+        self.vsa_api.delete(self.context, vsa_id)
+
+    @args('--id', dest='vsa_id', metavar="<vsa_id>",
+        help='VSA ID (optional)')
+    @args('--all', dest='all', action="store_true", default=False,
+        help='Show all available details')
+    @args('--drives', dest='drives', action="store_true",
+        help='Include drive-level details')
+    @args('--volumes', dest='volumes', action="store_true",
+        help='Include volume-level details')
+    @args('--instances', dest='instances', action="store_true",
+        help='Include instance-level details')
+    def list(self, vsa_id=None, all=False,
+             drives=False, volumes=False, instances=False):
+        """Describe all available VSAs (or particular one)."""
+
+        vsas = []
+        if vsa_id is not None:
+            internal_id = ec2utils.ec2_id_to_id(vsa_id)
+            vsa = self.vsa_api.get(self.context, internal_id)
+            vsas.append(vsa)
+        else:
+            vsas = self.vsa_api.get_all(self.context)
+
+        if all:
+            drives = volumes = instances = True
+
+        self._list(self.context, vsas, drives, volumes, instances)
+
+    def update_capabilities(self):
+        """Forces updates capabilities on all nova-volume nodes."""
+
+        rpc.fanout_cast(context.get_admin_context(),
+                 FLAGS.volume_topic,
+                 {"method": "notification",
+                  "args": {"event": "startup"}})
+
+
+class VsaDriveTypeCommands(object):
+    """Methods for dealing with VSA drive types"""
+
+    def __init__(self, *args, **kwargs):
+        super(VsaDriveTypeCommands, self).__init__(*args, **kwargs)
+        self.context = context.get_admin_context()
+        self._drive_type_template = '%s_%sGB_%sRPM'
+
+    def _list(self, drives):
+        format_str = "%-5s %-30s %-10s %-10s %-10s %-20s %-10s %s"
+        if len(drives):
+            print format_str %\
+                   (_('ID'),
+                    _('name'),
+                    _('type'),
+                    _('size_gb'),
+                    _('rpm'),
+                    _('capabilities'),
+                    _('visible'),
+                    _('createTime'))
+
+        for name, vol_type in drives.iteritems():
+            drive = vol_type.get('extra_specs')
+            print format_str %\
+                (str(vol_type['id']),
+                drive['drive_name'],
+                drive['drive_type'],
+                drive['drive_size'],
+                drive['drive_rpm'],
+                drive.get('capabilities', ''),
+                str(drive.get('visible', '')),
+                str(vol_type['created_at']))
+
+    @args('--type', dest='type', metavar="<type>",
+        help='Drive type (SATA, SAS, SSD, etc.)')
+    @args('--size', dest='size_gb', metavar="<gb>", help='Drive size in GB')
+    @args('--rpm', dest='rpm', metavar="<rpm>", help='RPM')
+    @args('--capabilities', dest='capabilities', default=None,
+        metavar="<string>", help='Different capabilities')
+    @args('--hide', dest='hide', action="store_true", default=False,
+        help='Show or hide drive')
+    @args('--name', dest='name', metavar="<name>", help='Drive name')
+    def create(self, type, size_gb, rpm, capabilities=None,
+                     hide=False, name=None):
+        """Create drive type."""
+
+        hide = True if hide in [True, "True", "--hide", "hide"] else False
+
+        if name is None:
+            name = self._drive_type_template % (type, size_gb, rpm)
+
+        extra_specs = {'type': 'vsa_drive',
+                       'drive_name': name,
+                       'drive_type': type,
+                       'drive_size': size_gb,
+                       'drive_rpm': rpm,
+                       'visible': True,
+                       }
+        if hide:
+            extra_specs['visible'] = False
+
+        if capabilities is not None and capabilities != '':
+            extra_specs['capabilities'] = capabilities
+
+        volume_types.create(self.context, name, extra_specs)
+        result = volume_types.get_volume_type_by_name(self.context, name)
+        self._list({name: result})
+
+    @args('--name', dest='name', metavar="<name>", help='Drive name')
+    @args('--purge', action="store_true", dest='purge', default=False,
+            help='purge record from database')
+    def delete(self, name, purge):
+        """Marks instance types / flavors as deleted"""
+        try:
+            if purge:
+                volume_types.purge(self.context, name)
+                verb = "purged"
+            else:
+                volume_types.destroy(self.context, name)
+                verb = "deleted"
+        except exception.ApiError:
+            print "Valid volume type name is required"
+            sys.exit(1)
+        except exception.DBError, e:
+            print "DB Error: %s" % e
+            sys.exit(2)
+        except:
+            sys.exit(3)
+        else:
+            print "%s %s" % (name, verb)
+
+    @args('--all', dest='all', action="store_true", default=False,
+        help='Show all drives (including invisible)')
+    @args('--name', dest='name', metavar="<name>",
+        help='Show only specified drive')
+    def list(self, all=False, name=None):
+        """Describe all available VSA drive types (or particular one)."""
+
+        all = False if all in ["--all", False, "False"] else True
+
+        search_opts = {'extra_specs': {'type': 'vsa_drive'}}
+        if name is not None:
+            search_opts['extra_specs']['name'] = name
+
+        if all == False:
+            search_opts['extra_specs']['visible'] = '1'
+
+        drives = volume_types.get_all_types(self.context,
+                                            search_opts=search_opts)
+        self._list(drives)
+
+    @args('--name', dest='name', metavar="<name>", help='Drive name')
+    @args('--type', dest='type', metavar="<type>",
+        help='Drive type (SATA, SAS, SSD, etc.)')
+    @args('--size', dest='size_gb', metavar="<gb>", help='Drive size in GB')
+    @args('--rpm', dest='rpm', metavar="<rpm>", help='RPM')
+    @args('--capabilities', dest='capabilities', default=None,
+        metavar="<string>", help='Different capabilities')
+    @args('--visible', dest='visible',
+        metavar="<show|hide>", help='Show or hide drive')
+    def update(self, name, type=None, size_gb=None, rpm=None,
+                     capabilities=None, visible=None):
+        """Update drive type."""
+
+        volume_type = volume_types.get_volume_type_by_name(self.context, name)
+
+        extra_specs = {'type': 'vsa_drive'}
+
+        if type:
+            extra_specs['drive_type'] = type
+
+        if size_gb:
+            extra_specs['drive_size'] = size_gb
+
+        if rpm:
+            extra_specs['drive_rpm'] = rpm
+
+        if capabilities:
+            extra_specs['capabilities'] = capabilities
+
+        if visible is not None:
+            if visible in ["show", True, "True"]:
+                extra_specs['visible'] = True
+            elif visible in ["hide", False, "False"]:
+                extra_specs['visible'] = False
+            else:
+                raise ValueError(_('visible parameter should be set to '\
+                                   'show or hide'))
+
+        db.api.volume_type_extra_specs_update_or_create(self.context,
+                    volume_type['id'],
+                    extra_specs)
+        result = volume_types.get_volume_type_by_name(self.context, name)
+        self._list({name: result})
+
+
 class VolumeCommands(object):
     """Methods for dealing with a cloud in an odd state"""
 
@@ -1483,6 +1957,7 @@ CATEGORIES = [
     ('agent', AgentBuildCommands),
     ('config', ConfigCommands),
     ('db', DbCommands),
+    ('drive', VsaDriveTypeCommands),
     ('fixed', FixedIpCommands),
     ('flavor', InstanceTypeCommands),
     ('floating', FloatingIpCommands),
@@ -1498,7 +1973,8 @@ CATEGORIES = [
     ('version', VersionCommands),
     ('vm', VmCommands),
     ('volume', VolumeCommands),
-    ('vpn', VpnCommands)]
+    ('vpn', VpnCommands),
+    ('vsa', VsaCommands)]
 
 
 def lazy_match(name, key_value_tuples):
diff --git a/bin/nova-vsa b/bin/nova-vsa
new file mode 100755
index 000000000..2d6eee2c0
--- /dev/null
+++ b/bin/nova-vsa
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright (c) 2011 Zadara Storage Inc.
+# Copyright (c) 2011 OpenStack LLC.
+#
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Starter script for Nova VSA."""
+
+import eventlet
+eventlet.monkey_patch()
+
+import os
+import sys
+
+# If ../nova/__init__.py exists, add ../ to Python search path, so that
+# it will override what happens to be installed in /usr/(local/)lib/python...
+possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
+                                   os.pardir,
+                                   os.pardir))
+if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')):
+    sys.path.insert(0, possible_topdir)
+
+
+from nova import flags
+from nova import log as logging
+from nova import service
+from nova import utils
+
+if __name__ == '__main__':
+    utils.default_flagfile()
+    flags.FLAGS(sys.argv)
+    logging.setup()
+    utils.monkey_patch()
+    server = service.Service.create(binary='nova-vsa')
+    service.serve(server)
+    service.wait()