diff options
166 files changed, 7132 insertions, 4698 deletions
@@ -19,11 +19,11 @@ <mordred@inaugust.com> <mordred@hudson> <paul@openstack.org> <pvoccio@castor.local> <paul@openstack.org> <paul.voccio@rackspace.com> +<soren.hansen@rackspace.com> <soren@linux2go.dk> <todd@ansolabs.com> <todd@lapex> <todd@ansolabs.com> <todd@rubidine.com> <vishvananda@gmail.com> <vishvananda@yahoo.com> <vishvananda@gmail.com> <root@mirror.nasanebula.net> -# These are from people who failed to set a proper committer -. <root@tonbuntu> -. <laner@controller> -. <root@ubuntu> +<vishvananda@gmail.com> <root@ubuntu> +<sleepsonthefloor@gmail.com> <root@tonbuntu> +<rlane@wikimedia.org> <laner@controller> @@ -3,8 +3,12 @@ Anne Gentle <anne@openstack.org> Anthony Young <sleepsonthefloor@gmail.com> Armando Migliaccio <Armando.Migliaccio@eu.citrix.com> Chris Behrens <cbehrens@codestud.com> +Chmouel Boudjnah <chmouel@chmouel.com> +David Pravec <David.Pravec@danix.org> Dean Troyer <dtroyer@gmail.com> Devin Carlen <devin.carlen@gmail.com> +Ed Leafe <ed@leafe.com> +Eldar Nugaev <enugaev@griddynamics.com> Eric Day <eday@oddments.org> Ewan Mellor <ewan.mellor@citrix.com> Hisaki Ohara <hisaki.ohara@intel.com> @@ -12,6 +16,7 @@ Jay Pipes <jaypipes@gmail.com> Jesse Andrews <anotherjesse@gmail.com> Joe Heck <heckj@mac.com> Joel Moore <joelbm24@gmail.com> +Jonathan Bryce <jbryce@jbryce.com> Josh Kearney <josh.kearney@rackspace.com> Joshua McKenty <jmckenty@gmail.com> Justin Santa Barbara <justin@fathomdb.com> @@ -20,8 +25,13 @@ Michael Gundlach <michael.gundlach@rackspace.com> Monty Taylor <mordred@inaugust.com> Paul Voccio <paul@openstack.org> Rick Clark <rick@openstack.org> +Ryan Lane <rlane@wikimedia.org> +Ryan Lucio <rlucio@internap.com> +Sandy Walsh <sandy.walsh@rackspace.com> Soren Hansen <soren.hansen@rackspace.com> +Thierry Carrez <thierry@openstack.org> Todd Willey <todd@ansolabs.com> +Trey Morris <trey.morris@rackspace.com> Vishvananda Ishaya <vishvananda@gmail.com> Youcef Laribi <Youcef.Laribi@eu.citrix.com> Zhixue Wu <Zhixue.Wu@citrix.com> diff --git a/CA/geninter.sh b/CA/geninter.sh index 7d6c280d5..1fbcc9e73 100755 --- a/CA/geninter.sh +++ b/CA/geninter.sh @@ -16,16 +16,24 @@ # License for the specific language governing permissions and limitations # under the License. -# ARG is the id of the user -export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-intCA-$1" -mkdir INTER/$1 -cd INTER/$1 +# $1 is the id of the project and $2 is the subject of the cert +NAME=$1 +SUBJ=$2 +mkdir -p projects/$NAME +cd projects/$NAME cp ../../openssl.cnf.tmpl openssl.cnf -sed -i -e s/%USERNAME%/$1/g openssl.cnf +sed -i -e s/%USERNAME%/$NAME/g openssl.cnf mkdir certs crl newcerts private +openssl req -new -x509 -extensions v3_ca -keyout private/cakey.pem -out cacert.pem -days 365 -config ./openssl.cnf -batch -nodes echo "10" > serial touch index.txt -openssl genrsa -out private/cakey.pem 1024 -config ./openssl.cnf -batch -nodes -openssl req -new -sha2 -key private/cakey.pem -out ../../reqs/inter$1.csr -batch -subj "$SUBJ" -cd ../../ -openssl ca -extensions v3_ca -days 365 -out INTER/$1/cacert.pem -in reqs/inter$1.csr -config openssl.cnf -batch +# NOTE(vish): Disabling intermediate ca's because we don't actually need them. +# It makes more sense to have each project have its own root ca. +# openssl genrsa -out private/cakey.pem 1024 -config ./openssl.cnf -batch -nodes +# openssl req -new -sha256 -key private/cakey.pem -out ../../reqs/inter$NAME.csr -batch -subj "$SUBJ" +openssl ca -gencrl -config ./openssl.cnf -out crl.pem +if [ "`id -u`" != "`grep nova /etc/passwd | cut -d':' -f3`" ]; then + sudo chown -R nova:nogroup . +fi +# cd ../../ +# openssl ca -extensions v3_ca -days 365 -out INTER/$NAME/cacert.pem -in reqs/inter$NAME.csr -config openssl.cnf -batch diff --git a/CA/genrootca.sh b/CA/genrootca.sh index 31976092e..8f2c3ee3f 100755 --- a/CA/genrootca.sh +++ b/CA/genrootca.sh @@ -25,4 +25,5 @@ else openssl req -new -x509 -extensions v3_ca -keyout private/cakey.pem -out cacert.pem -days 365 -config ./openssl.cnf -batch -nodes touch index.txt echo "10" > serial + openssl ca -gencrl -config ./openssl.cnf -out crl.pem fi diff --git a/nova/tests/validator_unittest.py b/CA/genvpn.sh index b5f1c0667..7e7db185d 100644..100755 --- a/nova/tests/validator_unittest.py +++ b/CA/genvpn.sh @@ -1,3 +1,4 @@ +#!/bin/bash # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2010 United States Government as represented by the @@ -16,27 +17,20 @@ # License for the specific language governing permissions and limitations # under the License. -import logging -import unittest +# This gets zipped and run on the cloudpipe-managed OpenVPN server +NAME=$1 +SUBJ=$2 -from nova import flags -from nova import test -from nova import validate +mkdir -p projects/$NAME +cd projects/$NAME +# generate a server priv key +openssl genrsa -out server.key 2048 -class ValidationTestCase(test.TrialTestCase): - def setUp(self): - super(ValidationTestCase, self).setUp() +# generate a server CSR +openssl req -new -key server.key -out server.csr -batch -subj "$SUBJ" - def tearDown(self): - super(ValidationTestCase, self).tearDown() - - def test_type_validation(self): - self.assertTrue(type_case("foo", 5, 1)) - self.assertRaises(TypeError, type_case, "bar", "5", 1) - self.assertRaises(TypeError, type_case, None, 5, 1) - - -@validate.typetest(instanceid=str, size=int, number_of_instances=int) -def type_case(instanceid, size, number_of_instances): - return True +novauid=`getent passwd nova | awk -F: '{print $3}'` +if [ ! -z "${novauid}" ] && [ "`id -u`" != "${novauid}" ]; then + sudo chown -R nova:nogroup . +fi diff --git a/CA/openssl.cnf.tmpl b/CA/openssl.cnf.tmpl index 639b8e80a..dd81f1c2b 100644 --- a/CA/openssl.cnf.tmpl +++ b/CA/openssl.cnf.tmpl @@ -24,7 +24,6 @@ dir = . [ ca ] default_ca = CA_default -unique_subject = no [ CA_default ] serial = $dir/serial @@ -32,6 +31,8 @@ database = $dir/index.txt new_certs_dir = $dir/newcerts certificate = $dir/cacert.pem private_key = $dir/private/cakey.pem +unique_subject = no +default_crl_days = 365 default_days = 365 default_md = md5 preserve = no diff --git a/CA/INTER/.gitignore b/CA/projects/.gitignore index 72e8ffc0d..72e8ffc0d 100644 --- a/CA/INTER/.gitignore +++ b/CA/projects/.gitignore diff --git a/CA/INTER/.placeholder b/CA/projects/.placeholder index e69de29bb..e69de29bb 100644 --- a/CA/INTER/.placeholder +++ b/CA/projects/.placeholder diff --git a/MANIFEST.in b/MANIFEST.in index 982b727aa..199ce30b6 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -13,7 +13,7 @@ include nova/cloudpipe/client.ovpn.template include nova/compute/fakevirtinstance.xml include nova/compute/interfaces.template include nova/virt/interfaces.template -include nova/virt/libvirt.*.xml.template +include nova/virt/libvirt*.xml.template include nova/tests/CA/ include nova/tests/CA/cacert.pem include nova/tests/CA/private/ diff --git a/README.livemigration.txt b/README.livemigration.txt index 9229bafa6..d859c24ca 100644 --- a/README.livemigration.txt +++ b/README.livemigration.txt @@ -147,7 +147,7 @@ 3. error checking - When live migration fails somehow, error message shows at: + When live migration fails somehow, error messages are shown at: a. scheduler logfile b. source compute node logfile c. dest compute node logfile diff --git a/bin/nova-api b/bin/nova-api index a9c53dbcd..1c671201e 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -17,10 +17,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -""" -Nova API daemon. -""" +"""Starter script for Nova API.""" + +import gettext import os import sys @@ -32,9 +32,13 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + +from nova import api from nova import flags from nova import utils -from nova import server +from nova import wsgi + FLAGS = flags.FLAGS flags.DEFINE_integer('osapi_port', 8774, 'OpenStack API port') @@ -43,15 +47,10 @@ flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port') flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host') -def main(_args): - from nova import api - from nova import wsgi +if __name__ == '__main__': + utils.default_flagfile() + FLAGS(sys.argv) server = wsgi.Server() server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host) server.start(api.API('ec2'), FLAGS.ec2api_port, host=FLAGS.ec2api_host) server.wait() - - -if __name__ == '__main__': - utils.default_flagfile() - server.serve('nova-api', main) diff --git a/bin/nova-combined b/bin/nova-combined new file mode 100755 index 000000000..53322f1a0 --- /dev/null +++ b/bin/nova-combined @@ -0,0 +1,68 @@ +#!/usr/bin/env python +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Combined starter script for Nova services.""" + +import eventlet +eventlet.monkey_patch() + +import gettext +import os +import sys + +# If ../nova/__init__.py exists, add ../ to Python search path, so that +# it will override what happens to be installed in /usr/(local/)lib/python... +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, + os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): + sys.path.insert(0, possible_topdir) + +gettext.install('nova', unicode=1) + +from nova import api +from nova import flags +from nova import service +from nova import utils +from nova import wsgi + + +FLAGS = flags.FLAGS +flags.DEFINE_integer('osapi_port', 8774, 'OpenStack API port') +flags.DEFINE_string('osapi_host', '0.0.0.0', 'OpenStack API host') +flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port') +flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host') + + +if __name__ == '__main__': + utils.default_flagfile() + FLAGS(sys.argv) + + compute = service.Service.create(binary='nova-compute') + network = service.Service.create(binary='nova-network') + volume = service.Service.create(binary='nova-volume') + scheduler = service.Service.create(binary='nova-scheduler') + #objectstore = service.Service.create(binary='nova-objectstore') + + service.serve(compute, network, volume, scheduler) + + server = wsgi.Server() + server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host) + server.start(api.API('ec2'), FLAGS.ec2api_port, host=FLAGS.ec2api_host) + server.wait() diff --git a/bin/nova-compute b/bin/nova-compute index ac6378f75..d2d352da2 100755 --- a/bin/nova-compute +++ b/bin/nova-compute @@ -17,10 +17,12 @@ # License for the specific language governing permissions and limitations # under the License. -""" - Twistd daemon for the nova compute nodes. -""" +"""Starter script for Nova Compute.""" +import eventlet +eventlet.monkey_patch() + +import gettext import os import sys @@ -32,14 +34,12 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import service -from nova import twistd from nova import utils - if __name__ == '__main__': utils.default_flagfile() - twistd.serve(__file__) - -if __name__ == '__builtin__': - application = service.Service.create() # pylint: disable=C0103 + service.serve() + service.wait() diff --git a/bin/nova-dhcpbridge b/bin/nova-dhcpbridge index 17c62da0a..828aba3d1 100755 --- a/bin/nova-dhcpbridge +++ b/bin/nova-dhcpbridge @@ -21,6 +21,7 @@ Handle lease database updates from DHCP servers. """ +import gettext import logging import os import sys @@ -33,6 +34,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import context from nova import db from nova import flags @@ -107,7 +110,6 @@ def main(): FLAGS.num_networks = 5 path = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', - '_trial_temp', 'nova.sqlite')) FLAGS.sql_connection = 'sqlite:///%s' % path action = argv[1] diff --git a/bin/nova-import-canonical-imagestore b/bin/nova-import-canonical-imagestore index 4ed9e8365..036b41e48 100755 --- a/bin/nova-import-canonical-imagestore +++ b/bin/nova-import-canonical-imagestore @@ -21,6 +21,7 @@ Download images from Canonical Image Store """ +import gettext import json import os import tempfile @@ -37,6 +38,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import flags from nova import utils from nova.objectstore import image diff --git a/bin/nova-instancemonitor b/bin/nova-instancemonitor index 9b6c40e82..5dac3ffe6 100755 --- a/bin/nova-instancemonitor +++ b/bin/nova-instancemonitor @@ -21,6 +21,7 @@ Daemon for Nova RRD based instance resource monitoring. """ +import gettext import os import logging import sys @@ -34,6 +35,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import utils from nova import twistd from nova.compute import monitor diff --git a/bin/nova-manage b/bin/nova-manage index d6aa29679..7c87d21ff 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -53,6 +53,7 @@ CLI interface for nova management. """ +import gettext import logging import os import sys @@ -68,20 +69,19 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import context +from nova import crypto from nova import db from nova import exception from nova import flags from nova import quota from nova import utils from nova.auth import manager -from nova.cloudpipe import pipelib -#added by masumotok from nova import rpc -# added by masumotok +from nova.cloudpipe import pipelib from nova.api.ec2 import cloud -# added by masumotok -from nova.compute import power_state @@ -100,47 +100,43 @@ class VpnCommands(object): self.manager = manager.AuthManager() self.pipe = pipelib.CloudPipe() - def list(self): - """Print a listing of the VPNs for all projects.""" + def list(self, project=None): + """Print a listing of the VPN data for one or all projects. + + args: [project=all]""" print "%-12s\t" % 'project', print "%-20s\t" % 'ip:port', + print "%-20s\t" % 'private_ip', print "%s" % 'state' - for project in self.manager.get_projects(): + if project: + projects = [self.manager.get_project(project)] + else: + projects = self.manager.get_projects() + # NOTE(vish): This hits the database a lot. We could optimize + # by getting all networks in one query and all vpns + # in aother query, then doing lookups by project + for project in projects: print "%-12s\t" % project.name, - - try: - s = "%s:%s" % (project.vpn_ip, project.vpn_port) - except exception.NotFound: - s = "None" - print "%-20s\t" % s, - - vpn = self._vpn_for(project.id) + ipport = "%s:%s" % (project.vpn_ip, project.vpn_port) + print "%-20s\t" % ipport, + ctxt = context.get_admin_context() + vpn = db.instance_get_project_vpn(ctxt, project.id) if vpn: - command = "ping -c1 -w1 %s > /dev/null; echo $?" - out, _err = utils.execute(command % vpn['private_dns_name'], - check_exit_code=False) - if out.strip() == '0': - net = 'up' - else: - net = 'down' - print vpn['private_dns_name'], - print vpn['node_name'], - print vpn['instance_id'], + address = None + state = 'down' + if vpn.get('fixed_ip', None): + address = vpn['fixed_ip']['address'] + if project.vpn_ip and utils.vpn_ping(project.vpn_ip, + project.vpn_port): + state = 'up' + print address, + print vpn['host'], + print vpn['ec2_id'], print vpn['state_description'], - print net - + print state else: print None - def _vpn_for(self, project_id): - """Get the VPN instance for a project ID.""" - for instance in db.instance_get_all(context.get_admin_context()): - if (instance['image_id'] == FLAGS.vpn_image_id - and not instance['state_description'] in - ['shutting_down', 'shutdown'] - and instance['project_id'] == project_id): - return instance - def spawn(self): """Run all VPNs.""" for p in reversed(self.manager.get_projects()): @@ -153,6 +149,21 @@ class VpnCommands(object): """Start the VPN for a given project.""" self.pipe.launch_vpn_instance(project_id) + def change(self, project_id, ip, port): + """Change the ip and port for a vpn. + + args: project, ip, port""" + project = self.manager.get_project(project_id) + if not project: + print 'No project %s' % (project_id) + return + admin = context.get_admin_context() + network_ref = db.project_get_network(admin, project_id) + db.network_update(admin, + network_ref['id'], + {'vpn_public_address': ip, + 'vpn_public_port': int(port)}) + class ShellCommands(object): def bpython(self): @@ -299,6 +310,14 @@ class UserCommands(object): is_admin = False self.manager.modify_user(name, access_key, secret_key, is_admin) + def revoke(self, user_id, project_id=None): + """revoke certs for a user + arguments: user_id [project_id]""" + if project_id: + crypto.revoke_certs_by_user_and_project(user_id, project_id) + else: + crypto.revoke_certs_by_user(user_id) + class ProjectCommands(object): """Class for managing projects.""" @@ -366,9 +385,14 @@ class ProjectCommands(object): def zipfile(self, project_id, user_id, filename='nova.zip'): """Exports credentials for project to a zip file arguments: project_id user_id [filename='nova.zip]""" - zip_file = self.manager.get_credentials(user_id, project_id) - with open(filename, 'w') as f: - f.write(zip_file) + try: + zip_file = self.manager.get_credentials(user_id, project_id) + with open(filename, 'w') as f: + f.write(zip_file) + except db.api.NoMoreNetworks: + print ('No more networks available. If this is a new ' + 'installation, you need\nto call something like this:\n\n' + ' nova-manage network create 10.0.0.0/8 10 64\n\n') class FloatingIpCommands(object): @@ -431,7 +455,7 @@ class NetworkCommands(object): int(network_size), int(vlan_start), int(vpn_start)) -# this class is added by masumotok + class InstanceCommands(object): """Class for mangaging VM instances.""" @@ -441,43 +465,29 @@ class InstanceCommands(object): logging.basicConfig() ctxt = context.get_admin_context() - # 1. whether destination host exists - host_ref = db.host_get_by_name(ctxt, dest) - - # 2. whether instance exists and running - # try-catch clause is necessary because only internal_id is shown - # when NotFound exception occurs. it isnot understandable to admins. - try : + try: internal_id = cloud.ec2_id_to_internal_id(ec2_id) instance_ref = db.instance_get_by_internal_id(ctxt, internal_id) - except exception.NotFound : - print 'Not found instance_id(%s (internal_id:%s))' % ( ec2_id, internal_id) - raise + instance_id = instance_ref['id'] + except exception.NotFound as e: + msg = _('instance(%s) is not found') + e.args += (msg % ec2_id,) + raise e - if power_state.RUNNING != instance_ref['state'] or \ - 'running' != instance_ref['state_description']: - raise exception.Invalid('Instance(%s) is not running' % ec2_id) - - # 3. the host where instance is running and dst host is not same - if dest == instance_ref['host'] : - msg = '%s is where %s is running now. choose other host.' % (dest, ec2_id) - raise exception.Invalid(msg) - - # 4. live migration ret = rpc.call(ctxt, FLAGS.scheduler_topic, - { "method": "live_migration", - "args": {"ec2_id": ec2_id, - "dest":dest}}) + {"method": "live_migration", + "args": {"instance_id": instance_id, + "dest": dest, + "topic": FLAGS.compute_topic}}) - if None != ret : + if None != ret: raise ret - print 'Finished all procedure. check instance are migrated successfully' + print 'Finished all procedure. Check migrating finishes successfully' print 'check status by using euca-describe-instances.' -# this class is created by masumotok class HostCommands(object): """Class for mangaging host(physical nodes).""" @@ -485,17 +495,18 @@ class HostCommands(object): def list(self): """describe host list.""" - # to supress msg: No handlers could be found for logger "amqplib" + # To supress msg: No handlers could be found for logger "amqplib" logging.basicConfig() host_refs = db.host_get_all(context.get_admin_context()) for host_ref in host_refs: print host_ref['name'] + def show(self, host): """describe cpu/memory/hdd info for host.""" - # to supress msg: No handlers could be found for logger "amqplib" + # To supress msg: No handlers could be found for logger "amqplib" logging.basicConfig() result = rpc.call(context.get_admin_context(), @@ -503,41 +514,27 @@ class HostCommands(object): {"method": "show_host_resource", "args": {"host": host}}) - # checing result msg format is necessary, that will have done + # Checking result msg format is necessary, that will have done # when this feture is included in API. if dict != type(result): print 'Unexpected error occurs' - elif not result['ret'] : + elif not result['ret']: print '%s' % result['msg'] - else : + else: cpu = result['phy_resource']['vcpus'] mem = result['phy_resource']['memory_mb'] hdd = result['phy_resource']['local_gb'] print 'HOST\t\tPROJECT\t\tcpu\tmem(mb)\tdisk(gb)' - print '%s\t\t\t%s\t%s\t%s' % ( host,cpu, mem, hdd) - for p_id, val in result['usage'].items() : - print '%s\t%s\t\t%s\t%s\t%s' % ( host, + print '%s\t\t\t%s\t%s\t%s' % (host, cpu, mem, hdd) + for p_id, val in result['usage'].items(): + print '%s\t%s\t\t%s\t%s\t%s' % (host, p_id, val['vcpus'], val['memory_mb'], val['local_gb']) - def has_keys(self, dic, keys): - not_found = [ key for key in keys if not dict.has_key(key) ] - return ( (0 == len(not_found)), not_found ) - - -# modified by masumotok -#CATEGORIES = [ -# ('user', UserCommands), -# ('project', ProjectCommands), -# ('role', RoleCommands), -# ('shell', ShellCommands), -# ('vpn', VpnCommands), -# ('floating', FloatingIpCommands), -# ('network', NetworkCommands)] CATEGORIES = [ ('user', UserCommands), ('project', ProjectCommands), @@ -547,7 +544,7 @@ CATEGORIES = [ ('floating', FloatingIpCommands), ('network', NetworkCommands), ('instance', InstanceCommands), - ('host',HostCommands)] + ('host', HostCommands)] def lazy_match(name, key_value_tuples): """Finds all objects that have a key that case insensitively contains diff --git a/bin/nova-network b/bin/nova-network index d1fb55261..0143846a7 100755 --- a/bin/nova-network +++ b/bin/nova-network @@ -17,10 +17,12 @@ # License for the specific language governing permissions and limitations # under the License. -""" - Twistd daemon for the nova network nodes. -""" +"""Starter script for Nova Network.""" +import eventlet +eventlet.monkey_patch() + +import gettext import os import sys @@ -32,14 +34,12 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import service -from nova import twistd from nova import utils - if __name__ == '__main__': utils.default_flagfile() - twistd.serve(__file__) - -if __name__ == '__builtin__': - application = service.Service.create() # pylint: disable-msg=C0103 + service.serve() + service.wait() diff --git a/bin/nova-objectstore b/bin/nova-objectstore index 00ae27af9..9fbe228a2 100755 --- a/bin/nova-objectstore +++ b/bin/nova-objectstore @@ -21,6 +21,7 @@ Twisted daemon for nova objectstore. Supports S3 API. """ +import gettext import os import sys @@ -32,6 +33,8 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import flags from nova import utils from nova import twistd diff --git a/bin/nova-scheduler b/bin/nova-scheduler index 4d1a40cf1..f4c0eaed6 100755 --- a/bin/nova-scheduler +++ b/bin/nova-scheduler @@ -17,10 +17,12 @@ # License for the specific language governing permissions and limitations # under the License. -""" - Twistd daemon for the nova scheduler nodes. -""" +"""Starter script for Nova Scheduler.""" +import eventlet +eventlet.monkey_patch() + +import gettext import os import sys @@ -32,14 +34,12 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import service -from nova import twistd from nova import utils - if __name__ == '__main__': utils.default_flagfile() - twistd.serve(__file__) - -if __name__ == '__builtin__': - application = service.Service.create() + service.serve() + service.wait() diff --git a/bin/nova-volume b/bin/nova-volume index e7281d6c0..ad3ddc405 100755 --- a/bin/nova-volume +++ b/bin/nova-volume @@ -17,10 +17,12 @@ # License for the specific language governing permissions and limitations # under the License. -""" - Twistd daemon for the nova volume nodes. -""" +"""Starter script for Nova Volume.""" +import eventlet +eventlet.monkey_patch() + +import gettext import os import sys @@ -32,14 +34,12 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): sys.path.insert(0, possible_topdir) +gettext.install('nova', unicode=1) + from nova import service -from nova import twistd from nova import utils - if __name__ == '__main__': utils.default_flagfile() - twistd.serve(__file__) - -if __name__ == '__builtin__': - application = service.Service.create() # pylint: disable-msg=C0103 + service.serve() + service.wait() diff --git a/contrib/nova.sh b/contrib/nova.sh index 1a9f93a3b..da1ba030c 100755 --- a/contrib/nova.sh +++ b/contrib/nova.sh @@ -15,13 +15,15 @@ if [ ! -n "$HOST_IP" ]; then # NOTE(vish): This will just get the first ip in the list, so if you # have more than one eth device set up, this will fail, and # you should explicitly set HOST_IP in your environment - HOST_IP=`ifconfig | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'` + HOST_IP=`LC_ALL=C ifconfig | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'` fi USE_MYSQL=${USE_MYSQL:-0} MYSQL_PASS=${MYSQL_PASS:-nova} TEST=${TEST:-0} USE_LDAP=${USE_LDAP:-0} +# Use OpenDJ instead of OpenLDAP when using LDAP +USE_OPENDJ=${USE_OPENDJ:-0} LIBVIRT_TYPE=${LIBVIRT_TYPE:-qemu} NET_MAN=${NET_MAN:-VlanManager} # NOTE(vish): If you are using FlatDHCP on multiple hosts, set the interface @@ -70,7 +72,7 @@ fi # You should only have to run this once if [ "$CMD" == "install" ]; then sudo apt-get install -y python-software-properties - sudo add-apt-repository ppa:nova-core/ppa + sudo add-apt-repository ppa:nova-core/trunk sudo apt-get update sudo apt-get install -y dnsmasq kpartx kvm gawk iptables ebtables sudo apt-get install -y user-mode-linux kvm libvirt-bin @@ -113,7 +115,13 @@ if [ "$CMD" == "run" ]; then rm $NOVA_DIR/nova.sqlite fi if [ "$USE_LDAP" == 1 ]; then - sudo $NOVA_DIR/nova/auth/slap.sh + if [ "$USE_OPENDJ" == 1 ]; then + echo '--ldap_user_dn=cn=Directory Manager' >> \ + /etc/nova/nova-manage.conf + sudo $NOVA_DIR/nova/auth/opendj.sh + else + sudo $NOVA_DIR/nova/auth/slap.sh + fi fi rm -rf $NOVA_DIR/instances mkdir -p $NOVA_DIR/instances diff --git a/contrib/puppet/files/production/nova-iptables b/contrib/puppet/files/production/nova-iptables index b7b52df87..61e2ca2b9 100755 --- a/contrib/puppet/files/production/nova-iptables +++ b/contrib/puppet/files/production/nova-iptables @@ -30,6 +30,8 @@ if [ -f /etc/default/nova-iptables ] ; then . /etc/default/nova-iptables fi +export LC_ALL=C + API_PORT=${API_PORT:-"8773"} if [ ! -n "$IP" ]; then diff --git a/doc/ext/nova_autodoc.py b/doc/ext/nova_autodoc.py index 39aa2c2cf..5429bb656 100644 --- a/doc/ext/nova_autodoc.py +++ b/doc/ext/nova_autodoc.py @@ -1,5 +1,8 @@ +import gettext import os +gettext.install('nova') + from nova import utils def setup(app): diff --git a/doc/source/adminguide/managing.networks.rst b/doc/source/adminguide/managing.networks.rst index b8563637e..38c1cba78 100644 --- a/doc/source/adminguide/managing.networks.rst +++ b/doc/source/adminguide/managing.networks.rst @@ -23,7 +23,7 @@ In Nova, users organize their cloud resources in projects. A Nova project consis Nova Network Strategies ----------------------- -Currently, Nova supports three kinds of networks, implemented in three "Network Manager" types respectively: Flat Network Manager, Flat DHCP Network Manager, and VLAN Network Manager. The three kinds of networks can c-exist in a cloud system. However, the scheduler for selecting the type of network for a given project is not yet implemented. Here is a brief description of each of the different network strategies, with a focus on the VLAN Manager in a separate section. +Currently, Nova supports three kinds of networks, implemented in three "Network Manager" types respectively: Flat Network Manager, Flat DHCP Network Manager, and VLAN Network Manager. The three kinds of networks can co-exist in a cloud system. However, the scheduler for selecting the type of network for a given project is not yet implemented. Here is a brief description of each of the different network strategies, with a focus on the VLAN Manager in a separate section. Read more about Nova network strategies here: diff --git a/doc/source/adminguide/multi.node.install.rst b/doc/source/adminguide/multi.node.install.rst index dcceb539b..fcb76c5e5 100644 --- a/doc/source/adminguide/multi.node.install.rst +++ b/doc/source/adminguide/multi.node.install.rst @@ -19,7 +19,7 @@ Installing Nova on Multiple Servers =================================== When you move beyond evaluating the technology and into building an actual -production environemnt, you will need to know how to configure your datacenter +production environment, you will need to know how to configure your datacenter and how to deploy components across your clusters. This guide should help you through that process. @@ -35,7 +35,6 @@ Requirements for a multi-node installation * For a recommended HA setup, consider a MySQL master/slave replication, with as many slaves as you like, and probably a heartbeat to kick one of the slaves into being a master if it dies. * For performance optimization, split reads and writes to the database. MySQL proxy is the easiest way to make this work if running MySQL. - Assumptions ^^^^^^^^^^^ @@ -69,14 +68,14 @@ Step 1 Use apt-get to get the latest code It is highly likely that there will be errors when the nova services come up since they are not yet configured. Don't worry, you're only at step 1! -Step 2 Setup configuration files (installed in /etc/nova) +Step 2 Setup configuration file (installed in /etc/nova) --------------------------------------------------------- Note: CC_ADDR=<the external IP address of your cloud controller> -1. These need to be defined in EACH configuration file +Nova development has consolidated all .conf files to nova.conf as of November 2010. References to specific .conf files may be ignored. -:: +#. These need to be defined in the nova.conf configuration file:: --sql_connection=mysql://root:nova@$CC_ADDR/nova # location of nova sql db --s3_host=$CC_ADDR # This is where nova is hosting the objectstore service, which @@ -87,31 +86,14 @@ Note: CC_ADDR=<the external IP address of your cloud controller> --ec2_url=http://$CC_ADDR:8773/services/Cloud --network_manager=nova.network.manager.FlatManager # simple, no-vlan networking type - -2. nova-manage specific flags - -:: - - --fixed_range=<network/prefix> # ip network to use for VM guests, ex 192.168.2.64/26 - --network_size=<# of addrs> # number of ip addrs to use for VM guests, ex 64 - - -3. nova-network specific flags - -:: - --fixed_range=<network/prefix> # ip network to use for VM guests, ex 192.168.2.64/26 --network_size=<# of addrs> # number of ip addrs to use for VM guests, ex 64 -4. Create a nova group - -:: +#. Create a nova group:: sudo addgroup nova -5. nova-objectstore specific flags < no specific config needed > - -Config files should be have their owner set to root:nova, and mode set to 0640, since they contain your MySQL server's root password. +The Nova config file should have its owner set to root:nova, and mode set to 0640, since they contain your MySQL server's root password. :: @@ -121,7 +103,7 @@ Config files should be have their owner set to root:nova, and mode set to 0640, Step 3 Setup the sql db ----------------------- -1. First you 'preseed' (using vishy's :doc:`../quickstart`). Run this as root. +1. First you 'preseed' (using the Quick Start method :doc:`../quickstart`). Run this as root. :: @@ -161,7 +143,7 @@ Step 3 Setup the sql db GRANT ALL PRIVILEGES ON *.* TO 'root'@'%' WITH GRANT OPTION; SET PASSWORD FOR 'root'@'%' = PASSWORD('nova'); -7. branch and install Nova +7. Branch and install Nova :: @@ -186,9 +168,7 @@ Step 4 Setup Nova environment Note: The nova-manage service assumes that the first IP address is your network (like 192.168.0.0), that the 2nd IP is your gateway (192.168.0.1), and that the broadcast is the very last IP in the range you defined (192.168.0.255). If this is not the case you will need to manually edit the sql db 'networks' table.o. -On running this command, entries are made in the 'networks' and 'fixed_ips' table. However, one of the networks listed in the 'networks' table needs to be marked as bridge in order for the code to know that a bridge exists. We ended up doing this manually, (update query fired directly in the DB). Is there a better way to mark a network as bridged? - -Update: This has been resolved w.e.f 27/10. network is marked as bridged automatically based on the type of n/w manager selected. +On running this command, entries are made in the 'networks' and 'fixed_ips' table. However, one of the networks listed in the 'networks' table needs to be marked as bridge in order for the code to know that a bridge exists. The Network is marked as bridged automatically based on the type of network manager selected. More networking details to create a network bridge for flat network ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -233,7 +213,6 @@ unzip them in your home directory, and add them to your environment:: echo ". creds/novarc" >> ~/.bashrc ~/.bashrc - Step 6 Restart all relevant services ------------------------------------ @@ -249,8 +228,8 @@ Restart relevant nova services:: .. todo:: do we still need the content below? -Bare-metal Provisioning ------------------------ +Bare-metal Provisioning Notes +----------------------------- To install the base operating system you can use PXE booting. diff --git a/doc/source/adminguide/single.node.install.rst b/doc/source/adminguide/single.node.install.rst index 27597962a..8572c5a4a 100644 --- a/doc/source/adminguide/single.node.install.rst +++ b/doc/source/adminguide/single.node.install.rst @@ -9,7 +9,7 @@ The fastest way to get a test cloud running is through our :doc:`../quickstart`. Step 1 and 2: Get the latest Nova code system software ------------------------------------------------------ -Depending on your system, the mehod for accomplishing this varies +Depending on your system, the method for accomplishing this varies .. toctree:: :maxdepth: 1 @@ -63,8 +63,20 @@ You see an access key and a secret key export, such as these made-up ones::: export EC2_ACCESS_KEY=4e6498a2-blah-blah-blah-17d1333t97fd export EC2_SECRET_KEY=0a520304-blah-blah-blah-340sp34k05bbe9a7 +Step 5: Create the network +-------------------------- + +Type or copy/paste in the following line to create a network prior to creating a project. + +:: + + sudo nova-manage network create 10.0.0.0/8 1 64 + +For this command, the IP address is the cidr notation for your netmask, such as 192.168.1.0/24. The value 1 is the total number of networks you want made, and the 64 value is the total number of ips in all networks. + +After running this command, entries are made in the 'networks' and 'fixed_ips' table in the database. -Step 5: Create a project with the user you created +Step 6: Create a project with the user you created -------------------------------------------------- Type or copy/paste in the following line to create a project named IRT (for Ice Road Truckers, of course) with the newly-created user named anne. @@ -94,7 +106,7 @@ Type or copy/paste in the following line to create a project named IRT (for Ice Data Base Updated -Step 6: Unzip the nova.zip +Step 7: Unzip the nova.zip -------------------------- You should have a nova.zip file in your current working directory. Unzip it with this command: @@ -116,7 +128,7 @@ You'll see these files extract. extracting: cacert.pem -Step 7: Source the rc file +Step 8: Source the rc file -------------------------- Type or copy/paste the following to source the novarc file in your current working directory. @@ -125,9 +137,9 @@ Type or copy/paste the following to source the novarc file in your current worki . novarc -Step 8: Pat yourself on the back :) +Step 9: Pat yourself on the back :) ----------------------------------- -Congratulations, your cloud is up and running, you’ve created an admin user, retrieved the user's credentials and put them in your environment. +Congratulations, your cloud is up and running, you’ve created an admin user, created a network, retrieved the user's credentials and put them in your environment. Now you need an image. diff --git a/doc/source/cloud101.rst b/doc/source/cloud101.rst index 87db5af1e..7c79d2a70 100644 --- a/doc/source/cloud101.rst +++ b/doc/source/cloud101.rst @@ -54,6 +54,8 @@ Cloud computing offers different service models depending on the capabilities a The US-based National Institute of Standards and Technology offers definitions for cloud computing and the service models that are emerging. +These definitions are summarized from http://csrc.nist.gov/groups/SNS/cloud-computing/. + SaaS - Software as a Service ^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -72,12 +74,15 @@ IaaS - Infrastructure as a Service Provides infrastructure such as computer instances, network connections, and storage so that people can run any software or operating system. -.. todo:: Use definitions from http://csrc.nist.gov/groups/SNS/cloud-computing/ and attribute NIST Types of Cloud Deployments -------------------------- -.. todo:: describe public/private/hybrid/etc +When you hear terms such as public cloud or private cloud, these refer to the deployment model for the cloud. A private cloud operates for a single organization, but can be managed on-premise or off-premise. A public cloud has an infrastructure that is available to the general public or a large industry group and is likely owned by a cloud services company. + +The NIST also defines community cloud as shared by several organizations supporting a specific community with shared concerns. + +A hybrid cloud can be a deployment model, as a composition of both public and private clouds, or a hybrid model for cloud computing may involve both virtual and physical servers. Work in the Clouds ------------------ diff --git a/doc/source/images/novascreens.png b/doc/source/images/novascreens.png Binary files differnew file mode 100644 index 000000000..0fe3279cf --- /dev/null +++ b/doc/source/images/novascreens.png diff --git a/doc/source/images/novashvirtually.png b/doc/source/images/novashvirtually.png Binary files differnew file mode 100644 index 000000000..02c7e767c --- /dev/null +++ b/doc/source/images/novashvirtually.png diff --git a/doc/source/index.rst b/doc/source/index.rst index 9b2c8e1f8..b9ba6208a 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -26,7 +26,7 @@ Nova is written with the following design guidelines in mind: * **Component based architecture**: Quickly add new behaviors * **Highly available**: Scale to very serious workloads -* **Fault-Tollerant**: Isloated processes avoid cascading failures +* **Fault-Tolerant**: Isolated processes avoid cascading failures * **Recoverable**: Failures should be easy to diagnose, debug, and rectify * **Open Standards**: Be a reference implementation for a community-driven api * **API Compatibility**: Nova strives to provide API-compatible with popular systems like Amazon EC2 @@ -62,8 +62,6 @@ Administrator's Documentation adminguide/single.node.install adminguide/multi.node.install -.. todo:: add swiftadmin - Developer Docs ============== diff --git a/doc/source/livecd.rst b/doc/source/livecd.rst index 82cf4658a..b355fa180 100644 --- a/doc/source/livecd.rst +++ b/doc/source/livecd.rst @@ -1,2 +1,48 @@ Installing the Live CD ====================== + +If you'd like to set up a sandbox installation of Nova, you can use one of these Live CD images. + +If you don't already have VirtualBox installed, you can download it from http://www.virtualbox.org/wiki/Downloads. + +Download the zip or iso file and then follow these steps to try Nova in a virtual environment. + +http://c0047913.cdn1.cloudfiles.rackspacecloud.com/OpenStackNova.x86_64-2010.1.2.iso (OpenSUSE image; root password is "linux" for this image) + +http://c0028699.cdn1.cloudfiles.rackspacecloud.com/nova-vm.zip (~900 MB) (log in information is nova/nova) + +Once a VM is configured and started, here are the basics: + + #. Login to Ubuntu using ID nova and Password nova. + + #. Switch to running as sudo (enter nova when prompted for the password):: + + sudo -s + + #. To run Nova for the first time, enter:: + + cd /var/openstack/ + + #. Now that you're in the correct directory, enter:: + + ./nova.sh run + + .. image:: images/novashvirtually.png + +If it's already running, use screen -ls, and when the nova screen is presented,then enter screen -d -r nova. + +These are the steps to get an instance running (the image is already provided in this environment). Enter these commands in the "test" screen. + +:: + + euca-add-keypair test > test.pem + chmod 600 test.pem + euca-run-instances -k test -t m1.tiny ami-tiny + euca-describe-instances + + ssh -i test.pem root@10.0.0.3 + +To see output from the various workers, switch screen windows with Ctrl+A " (quotation mark). + + .. image:: images/novascreens.png + diff --git a/nova/adminclient.py b/nova/adminclient.py index 5a62cce7d..6ae9f0c0f 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -194,6 +194,7 @@ class HostInfo(object): class NovaAdminClient(object): + def __init__( self, clc_url=DEFAULT_CLC_URL, diff --git a/nova/api/__init__.py b/nova/api/__init__.py index 80f9f2109..26fed847b 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -24,14 +24,13 @@ Root WSGI middleware for all API controllers. :ec2api_subdomain: subdomain running the EC2 API (default: ec2) """ +import logging import routes import webob.dec from nova import flags -from nova import utils from nova import wsgi -from nova.api import cloudpipe from nova.api import ec2 from nova.api import openstack from nova.api.ec2 import metadatarequesthandler @@ -41,6 +40,7 @@ flags.DEFINE_string('osapi_subdomain', 'api', 'subdomain running the OpenStack API') flags.DEFINE_string('ec2api_subdomain', 'ec2', 'subdomain running the EC2 API') + FLAGS = flags.FLAGS @@ -80,7 +80,6 @@ class API(wsgi.Router): mapper.connect('%s/{path_info:.*}' % s, controller=mrh, conditions=ec2api_subdomain) - mapper.connect("/cloudpipe/{path_info:.*}", controller=cloudpipe.API()) super(API, self).__init__(mapper) @webob.dec.wsgify diff --git a/nova/api/cloud.py b/nova/api/cloud.py deleted file mode 100644 index b8f15019f..000000000 --- a/nova/api/cloud.py +++ /dev/null @@ -1,58 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Methods for API calls to control instances via AMQP. -""" - - -from nova import db -from nova import flags -from nova import rpc - -FLAGS = flags.FLAGS - - -def reboot(instance_id, context=None): - """Reboot the given instance.""" - instance_ref = db.instance_get_by_internal_id(context, instance_id) - host = instance_ref['host'] - rpc.cast(context, - db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "reboot_instance", - "args": {"instance_id": instance_ref['id']}}) - - -def rescue(instance_id, context): - """Rescue the given instance.""" - instance_ref = db.instance_get_by_internal_id(context, instance_id) - host = instance_ref['host'] - rpc.cast(context, - db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "rescue_instance", - "args": {"instance_id": instance_ref['id']}}) - - -def unrescue(instance_id, context): - """Unrescue the given instance.""" - instance_ref = db.instance_get_by_internal_id(context, instance_id) - host = instance_ref['host'] - rpc.cast(context, - db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "unrescue_instance", - "args": {"instance_id": instance_ref['id']}}) diff --git a/nova/api/cloudpipe/__init__.py b/nova/api/cloudpipe/__init__.py deleted file mode 100644 index 6d40990a8..000000000 --- a/nova/api/cloudpipe/__init__.py +++ /dev/null @@ -1,69 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -REST API Request Handlers for CloudPipe -""" - -import logging -import urllib -import webob -import webob.dec -import webob.exc - -from nova import crypto -from nova import wsgi -from nova.auth import manager -from nova.api.ec2 import cloud - - -_log = logging.getLogger("api") -_log.setLevel(logging.DEBUG) - - -class API(wsgi.Application): - - def __init__(self): - self.controller = cloud.CloudController() - - @webob.dec.wsgify - def __call__(self, req): - if req.method == 'POST': - return self.sign_csr(req) - _log.debug("Cloudpipe path is %s" % req.path_info) - if req.path_info.endswith("/getca/"): - return self.send_root_ca(req) - return webob.exc.HTTPNotFound() - - def get_project_id_from_ip(self, ip): - # TODO(eday): This was removed with the ORM branch, fix! - instance = self.controller.get_instance_by_ip(ip) - return instance['project_id'] - - def send_root_ca(self, req): - _log.debug("Getting root ca") - project_id = self.get_project_id_from_ip(req.remote_addr) - res = webob.Response() - res.headers["Content-Type"] = "text/plain" - res.body = crypto.fetch_ca(project_id) - return res - - def sign_csr(self, req): - project_id = self.get_project_id_from_ip(req.remote_addr) - cert = self.str_params['cert'] - return crypto.sign_csr(urllib.unquote(cert), project_id) diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index a6ee16c33..51d33bcc6 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -26,8 +26,8 @@ import webob import webob.dec import webob.exc -from nova import exception from nova import context +from nova import exception from nova import flags from nova import wsgi from nova.api.ec2 import apirequest @@ -37,16 +37,82 @@ from nova.auth import manager FLAGS = flags.FLAGS +flags.DEFINE_boolean('use_forwarded_for', False, + 'Treat X-Forwarded-For as the canonical remote address. ' + 'Only enable this if you have a sanitizing proxy.') +flags.DEFINE_boolean('use_lockout', False, + 'Whether or not to use lockout middleware.') +flags.DEFINE_integer('lockout_attempts', 5, + 'Number of failed auths before lockout.') +flags.DEFINE_integer('lockout_minutes', 15, + 'Number of minutes to lockout if triggered.') +flags.DEFINE_integer('lockout_window', 15, + 'Number of minutes for lockout window.') +flags.DEFINE_list('lockout_memcached_servers', None, + 'Memcached servers or None for in process cache.') + + _log = logging.getLogger("api") _log.setLevel(logging.DEBUG) class API(wsgi.Middleware): - """Routing for all EC2 API requests.""" def __init__(self): self.application = Authenticate(Router(Authorizer(Executor()))) + if FLAGS.use_lockout: + self.application = Lockout(self.application) + + +class Lockout(wsgi.Middleware): + """Lockout for x minutes on y failed auths in a z minute period. + + x = lockout_timeout flag + y = lockout_window flag + z = lockout_attempts flag + + Uses memcached if lockout_memcached_servers flag is set, otherwise it + uses a very simple in-proccess cache. Due to the simplicity of + the implementation, the timeout window is started with the first + failed request, so it will block if there are x failed logins within + that period. + + There is a possible race condition where simultaneous requests could + sneak in before the lockout hits, but this is extremely rare and would + only result in a couple of extra failed attempts.""" + + def __init__(self, application): + """middleware can use fake for testing.""" + if FLAGS.lockout_memcached_servers: + import memcache + else: + from nova import fakememcache as memcache + self.mc = memcache.Client(FLAGS.lockout_memcached_servers, + debug=0) + super(Lockout, self).__init__(application) + + @webob.dec.wsgify + def __call__(self, req): + access_key = str(req.params['AWSAccessKeyId']) + failures_key = "authfailures-%s" % access_key + failures = int(self.mc.get(failures_key) or 0) + if failures >= FLAGS.lockout_attempts: + detail = "Too many failed authentications." + raise webob.exc.HTTPForbidden(detail=detail) + res = req.get_response(self.application) + if res.status_int == 403: + failures = self.mc.incr(failures_key) + if failures is None: + # NOTE(vish): To use incr, failures has to be a string. + self.mc.set(failures_key, '1', time=FLAGS.lockout_window * 60) + elif failures >= FLAGS.lockout_attempts: + _log.warn('Access key %s has had %d failed authentications' + ' and will be locked out for %d minutes.' % + (access_key, failures, FLAGS.lockout_minutes)) + self.mc.set(failures_key, str(failures), + time=FLAGS.lockout_minutes * 60) + return res class Authenticate(wsgi.Middleware): @@ -77,13 +143,16 @@ class Authenticate(wsgi.Middleware): req.host, req.path) except exception.Error, ex: - logging.debug("Authentication Failure: %s" % ex) + logging.debug(_("Authentication Failure: %s") % ex) raise webob.exc.HTTPForbidden() # Authenticated! + remote_address = req.remote_addr + if FLAGS.use_forwarded_for: + remote_address = req.headers.get('X-Forwarded-For', remote_address) ctxt = context.RequestContext(user=user, project=project, - remote_address=req.remote_addr) + remote_address=remote_address) req.environ['ec2.context'] = ctxt return self.application @@ -120,9 +189,9 @@ class Router(wsgi.Middleware): except: raise webob.exc.HTTPBadRequest() - _log.debug('action: %s' % action) + _log.debug(_('action: %s') % action) for key, value in args.items(): - _log.debug('arg: %s\t\tval: %s' % (key, value)) + _log.debug(_('arg: %s\t\tval: %s') % (key, value)) # Success! req.environ['ec2.controller'] = controller diff --git a/nova/api/ec2/admin.py b/nova/api/ec2/admin.py index 1c6ab688d..fac01369e 100644 --- a/nova/api/ec2/admin.py +++ b/nova/api/ec2/admin.py @@ -168,6 +168,7 @@ class AdminController(object): # FIXME(vish): these host commands don't work yet, perhaps some of the # required data can be retrieved from service objects? + def describe_hosts(self, _context, **_kwargs): """Returns status info for all nodes. Includes: * Disk Space diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py index 5758781b6..a90fbeb0c 100644 --- a/nova/api/ec2/apirequest.py +++ b/nova/api/ec2/apirequest.py @@ -92,8 +92,8 @@ class APIRequest(object): method = getattr(self.controller, _camelcase_to_underscore(self.action)) except AttributeError: - _error = ('Unsupported API request: controller = %s,' - 'action = %s') % (self.controller, self.action) + _error = _('Unsupported API request: controller = %s,' + 'action = %s') % (self.controller, self.action) _log.warning(_error) # TODO: Raise custom exception, trap in apiserver, # and reraise as 400 error. diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index ebf5bcf0b..3d88e5ef3 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -27,7 +27,6 @@ import datetime import logging import re import os -import time from nova import context import IPy @@ -41,8 +40,6 @@ from nova import rpc from nova import utils from nova.compute import api as compute_api from nova.compute import instance_types -from nova.api import cloud -from nova.image.s3 import S3ImageService FLAGS = flags.FLAGS @@ -95,8 +92,9 @@ class CloudController(object): """ def __init__(self): self.network_manager = utils.import_object(FLAGS.network_manager) - self.compute_api = compute_api.ComputeAPI() - self.image_service = S3ImageService() + self.image_service = utils.import_object(FLAGS.image_service) + self.compute_api = compute_api.ComputeAPI(self.network_manager, + self.image_service) self.setup() def __str__(self): @@ -115,12 +113,12 @@ class CloudController(object): start = os.getcwd() os.chdir(FLAGS.ca_path) # TODO(vish): Do this with M2Crypto instead - utils.runthis("Generating root CA: %s", "sh genrootca.sh") + utils.runthis(_("Generating root CA: %s"), "sh genrootca.sh") os.chdir(start) def _get_mpi_data(self, context, project_id): result = {} - for instance in db.instance_get_all_by_project(context, project_id): + for instance in self.compute_api.get_instances(context, project_id): if instance['fixed_ip']: line = '%s slots=%d' % (instance['fixed_ip']['address'], instance['vcpus']) @@ -197,15 +195,19 @@ class CloudController(object): if FLAGS.region_list: regions = [] for region in FLAGS.region_list: - name, _sep, url = region.partition('=') + name, _sep, host = region.partition('=') + endpoint = '%s://%s:%s%s' % (FLAGS.ec2_prefix, + host, + FLAGS.cc_port, + FLAGS.ec2_suffix) regions.append({'regionName': name, - 'regionEndpoint': url}) + 'regionEndpoint': endpoint}) else: regions = [{'regionName': 'nova', - 'regionEndpoint': FLAGS.ec2_url}] - if region_name: - regions = [r for r in regions if r['regionName'] in region_name] - return {'regionInfo': regions} + 'regionEndpoint': '%s://%s:%s%s' % (FLAGS.ec2_prefix, + FLAGS.cc_host, + FLAGS.cc_port, + FLAGS.ec2_suffix)}] def describe_snapshots(self, context, @@ -319,11 +321,11 @@ class CloudController(object): ip_protocol = str(ip_protocol) if ip_protocol.upper() not in ['TCP', 'UDP', 'ICMP']: - raise InvalidInputException('%s is not a valid ipProtocol' % + raise InvalidInputException(_('%s is not a valid ipProtocol') % (ip_protocol,)) if ((min(from_port, to_port) < -1) or (max(from_port, to_port) > 65535)): - raise InvalidInputException('Invalid port range') + raise InvalidInputException(_('Invalid port range')) values['protocol'] = ip_protocol values['from_port'] = from_port @@ -361,7 +363,8 @@ class CloudController(object): criteria = self._revoke_rule_args_to_dict(context, **kwargs) if criteria == None: - raise exception.ApiError("No rule for the specified parameters.") + raise exception.ApiError(_("No rule for the specified " + "parameters.")) for rule in security_group.rules: match = True @@ -372,7 +375,7 @@ class CloudController(object): db.security_group_rule_destroy(context, rule['id']) self._trigger_refresh_security_group(context, security_group) return True - raise exception.ApiError("No rule for the specified parameters.") + raise exception.ApiError(_("No rule for the specified parameters.")) # TODO(soren): This has only been tested with Boto as the client. # Unfortunately, it seems Boto is using an old API @@ -388,8 +391,8 @@ class CloudController(object): values['parent_group_id'] = security_group.id if self._security_group_rule_exists(security_group, values): - raise exception.ApiError('This rule already exists in group %s' % - group_name) + raise exception.ApiError(_('This rule already exists in group %s') + % group_name) security_group_rule = db.security_group_rule_create(context, values) @@ -417,7 +420,7 @@ class CloudController(object): def create_security_group(self, context, group_name, group_description): self.compute_api.ensure_default_security_group(context) if db.security_group_exists(context, context.project_id, group_name): - raise exception.ApiError('group %s already exists' % group_name) + raise exception.ApiError(_('group %s already exists') % group_name) group = {'user_id': context.user.id, 'project_id': context.project_id, @@ -439,7 +442,7 @@ class CloudController(object): # instance_id is passed in as a list of instances ec2_id = instance_id[0] internal_id = ec2_id_to_internal_id(ec2_id) - instance_ref = db.instance_get_by_internal_id(context, internal_id) + instance_ref = self.compute_api.get_instance(context, internal_id) output = rpc.call(context, '%s.%s' % (FLAGS.compute_topic, instance_ref['host']), @@ -451,13 +454,15 @@ class CloudController(object): "Timestamp": now, "output": base64.b64encode(output)} - def describe_volumes(self, context, **kwargs): + def describe_volumes(self, context, volume_id=None, **kwargs): if context.user.is_admin(): volumes = db.volume_get_all(context) else: volumes = db.volume_get_all_by_project(context, context.project_id) - volumes = [self._format_volume(context, v) for v in volumes] + # NOTE(vish): volume_id is an optional list of volume ids to filter by. + volumes = [self._format_volume(context, v) for v in volumes + if volume_id is None or v['ec2_id'] in volume_id] return {'volumeSet': volumes} @@ -528,15 +533,15 @@ class CloudController(object): def attach_volume(self, context, volume_id, instance_id, device, **kwargs): volume_ref = db.volume_get_by_ec2_id(context, volume_id) if not re.match("^/dev/[a-z]d[a-z]+$", device): - raise exception.ApiError("Invalid device specified: %s. " - "Example device: /dev/vdb" % device) + raise exception.ApiError(_("Invalid device specified: %s. " + "Example device: /dev/vdb") % device) # TODO(vish): abstract status checking? if volume_ref['status'] != "available": - raise exception.ApiError("Volume status must be available") + raise exception.ApiError(_("Volume status must be available")) if volume_ref['attach_status'] == "attached": - raise exception.ApiError("Volume is already attached") + raise exception.ApiError(_("Volume is already attached")) internal_id = ec2_id_to_internal_id(instance_id) - instance_ref = db.instance_get_by_internal_id(context, internal_id) + instance_ref = self.compute_api.get_instance(context, internal_id) host = instance_ref['host'] rpc.cast(context, db.queue_get_for(context, FLAGS.compute_topic, host), @@ -556,10 +561,10 @@ class CloudController(object): instance_ref = db.volume_get_instance(context.elevated(), volume_ref['id']) if not instance_ref: - raise exception.ApiError("Volume isn't attached to anything!") + raise exception.ApiError(_("Volume isn't attached to anything!")) # TODO(vish): abstract status checking? if volume_ref['status'] == "available": - raise exception.ApiError("Volume is already detached") + raise exception.ApiError(_("Volume is already detached")) try: host = instance_ref['host'] rpc.cast(context, @@ -614,11 +619,7 @@ class CloudController(object): instances = db.instance_get_all_by_reservation(context, reservation_id) else: - if context.user.is_admin(): - instances = db.instance_get_all(context) - else: - instances = db.instance_get_all_by_project(context, - context.project_id) + instances = self.compute_api.get_instances(context) for instance in instances: if not context.user.is_admin(): if instance['image_id'] == FLAGS.vpn_image_id: @@ -701,34 +702,43 @@ class CloudController(object): def allocate_address(self, context, **kwargs): # check quota if quota.allowed_floating_ips(context, 1) < 1: - logging.warn("Quota exceeeded for %s, tried to allocate address", + logging.warn(_("Quota exceeeded for %s, tried to allocate " + "address"), context.project_id) - raise quota.QuotaError("Address quota exceeded. You cannot " - "allocate any more addresses") - network_topic = self._get_network_topic(context) + raise quota.QuotaError(_("Address quota exceeded. You cannot " + "allocate any more addresses")) + # NOTE(vish): We don't know which network host should get the ip + # when we allocate, so just send it to any one. This + # will probably need to move into a network supervisor + # at some point. public_ip = rpc.call(context, - network_topic, + FLAGS.network_topic, {"method": "allocate_floating_ip", "args": {"project_id": context.project_id}}) return {'addressSet': [{'publicIp': public_ip}]} def release_address(self, context, public_ip, **kwargs): - # NOTE(vish): Should we make sure this works? floating_ip_ref = db.floating_ip_get_by_address(context, public_ip) - network_topic = self._get_network_topic(context) + # NOTE(vish): We don't know which network host should get the ip + # when we deallocate, so just send it to any one. This + # will probably need to move into a network supervisor + # at some point. rpc.cast(context, - network_topic, + FLAGS.network_topic, {"method": "deallocate_floating_ip", "args": {"floating_address": floating_ip_ref['address']}}) return {'releaseResponse': ["Address released."]} def associate_address(self, context, instance_id, public_ip, **kwargs): internal_id = ec2_id_to_internal_id(instance_id) - instance_ref = db.instance_get_by_internal_id(context, internal_id) + instance_ref = self.compute_api.get_instance(context, internal_id) fixed_address = db.instance_get_fixed_address(context, instance_ref['id']) floating_ip_ref = db.floating_ip_get_by_address(context, public_ip) - network_topic = self._get_network_topic(context) + # NOTE(vish): Perhaps we should just pass this on to compute and + # let compute communicate with network. + network_topic = self.compute_api.get_network_topic(context, + internal_id) rpc.cast(context, network_topic, {"method": "associate_floating_ip", @@ -738,39 +748,31 @@ class CloudController(object): def disassociate_address(self, context, public_ip, **kwargs): floating_ip_ref = db.floating_ip_get_by_address(context, public_ip) - network_topic = self._get_network_topic(context) + # NOTE(vish): Get the topic from the host name of the network of + # the associated fixed ip. + if not floating_ip_ref.get('fixed_ip'): + raise exception.ApiError('Address is not associated.') + host = floating_ip_ref['fixed_ip']['network']['host'] + topic = db.queue_get_for(context, FLAGS.network_topic, host) rpc.cast(context, - network_topic, + topic, {"method": "disassociate_floating_ip", "args": {"floating_address": floating_ip_ref['address']}}) return {'disassociateResponse': ["Address disassociated."]} - def _get_network_topic(self, context): - """Retrieves the network host for a project""" - network_ref = self.network_manager.get_network(context) - host = network_ref['host'] - if not host: - host = rpc.call(context, - FLAGS.network_topic, - {"method": "set_network_host", - "args": {"network_id": network_ref['id']}}) - return db.queue_get_for(context, FLAGS.network_topic, host) - def run_instances(self, context, **kwargs): max_count = int(kwargs.get('max_count', 1)) instances = self.compute_api.create_instances(context, instance_types.get_by_type(kwargs.get('instance_type', None)), - self.image_service, kwargs['image_id'], - self._get_network_topic(context), min_count=int(kwargs.get('min_count', max_count)), max_count=max_count, - kernel_id=kwargs.get('kernel_id'), + kernel_id=kwargs.get('kernel_id', None), ramdisk_id=kwargs.get('ramdisk_id'), - name=kwargs.get('display_name'), + display_name=kwargs.get('display_name'), description=kwargs.get('display_description'), - user_data=kwargs.get('user_data', ''), key_name=kwargs.get('key_name'), + user_data=kwargs.get('user_data'), security_group=kwargs.get('security_group'), generate_hostname=internal_id_to_ec2_id) return self._format_run_instances(context, @@ -778,84 +780,30 @@ class CloudController(object): def terminate_instances(self, context, instance_id, **kwargs): """Terminate each instance in instance_id, which is a list of ec2 ids. - - instance_id is a kwarg so its name cannot be modified. - """ - ec2_id_list = instance_id + instance_id is a kwarg so its name cannot be modified.""" logging.debug("Going to start terminating instances") - for id_str in ec2_id_list: - internal_id = ec2_id_to_internal_id(id_str) - logging.debug("Going to try and terminate %s" % id_str) - try: - instance_ref = db.instance_get_by_internal_id(context, - internal_id) - except exception.NotFound: - logging.warning("Instance %s was not found during terminate", - id_str) - continue - - if (instance_ref['state_description'] == 'terminating'): - logging.warning("Instance %s is already being terminated", - id_str) - continue - now = datetime.datetime.utcnow() - self.compute_api.update_instance(context, - instance_ref['id'], - state_description='terminating', - state=0, - terminated_at=now) - - # FIXME(ja): where should network deallocate occur? - address = db.instance_get_floating_address(context, - instance_ref['id']) - if address: - logging.debug("Disassociating address %s" % address) - # NOTE(vish): Right now we don't really care if the ip is - # disassociated. We may need to worry about - # checking this later. Perhaps in the scheduler? - network_topic = self._get_network_topic(context) - rpc.cast(context, - network_topic, - {"method": "disassociate_floating_ip", - "args": {"floating_address": address}}) - - address = db.instance_get_fixed_address(context, - instance_ref['id']) - if address: - logging.debug("Deallocating address %s" % address) - # NOTE(vish): Currently, nothing needs to be done on the - # network node until release. If this changes, - # we will need to cast here. - self.network_manager.deallocate_fixed_ip(context.elevated(), - address) - - host = instance_ref['host'] - if host: - rpc.cast(context, - db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": "terminate_instance", - "args": {"instance_id": instance_ref['id']}}) - else: - db.instance_destroy(context, instance_ref['id']) + for ec2_id in instance_id: + internal_id = ec2_id_to_internal_id(ec2_id) + self.compute_api.delete_instance(context, internal_id) return True def reboot_instances(self, context, instance_id, **kwargs): """instance_id is a list of instance ids""" for ec2_id in instance_id: internal_id = ec2_id_to_internal_id(ec2_id) - cloud.reboot(internal_id, context=context) + self.compute_api.reboot(context, internal_id) return True def rescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" internal_id = ec2_id_to_internal_id(instance_id) - cloud.rescue(internal_id, context=context) + self.compute_api.rescue(context, internal_id) return True def unrescue_instance(self, context, instance_id, **kwargs): """This is an extension to the normal ec2_api""" internal_id = ec2_id_to_internal_id(instance_id) - cloud.unrescue(internal_id, context=context) + self.compute_api.unrescue(context, internal_id) return True def update_instance(self, context, ec2_id, **kwargs): @@ -866,7 +814,7 @@ class CloudController(object): changes[field] = kwargs[field] if changes: internal_id = ec2_id_to_internal_id(ec2_id) - inst = db.instance_get_by_internal_id(context, internal_id) + inst = self.compute_api.get_instance(context, internal_id) db.instance_update(context, inst['id'], kwargs) return True @@ -874,7 +822,7 @@ class CloudController(object): # TODO: return error if not authorized volume_ref = db.volume_get_by_ec2_id(context, volume_id) if volume_ref['status'] != "available": - raise exception.ApiError("Volume status must be available") + raise exception.ApiError(_("Volume status must be available")) now = datetime.datetime.utcnow() db.volume_update(context, volume_ref['id'], {'status': 'deleting', 'terminated_at': now}) @@ -905,11 +853,12 @@ class CloudController(object): def describe_image_attribute(self, context, image_id, attribute, **kwargs): if attribute != 'launchPermission': - raise exception.ApiError('attribute not supported: %s' % attribute) + raise exception.ApiError(_('attribute not supported: %s') + % attribute) try: image = self.image_service.show(context, image_id) except IndexError: - raise exception.ApiError('invalid id: %s' % image_id) + raise exception.ApiError(_('invalid id: %s') % image_id) result = {'image_id': image_id, 'launchPermission': []} if image['isPublic']: result['launchPermission'].append({'group': 'all'}) @@ -919,13 +868,14 @@ class CloudController(object): operation_type, **kwargs): # TODO(devcamcar): Support users and groups other than 'all'. if attribute != 'launchPermission': - raise exception.ApiError('attribute not supported: %s' % attribute) + raise exception.ApiError(_('attribute not supported: %s') + % attribute) if not 'user_group' in kwargs: - raise exception.ApiError('user or group not specified') + raise exception.ApiError(_('user or group not specified')) if len(kwargs['user_group']) != 1 and kwargs['user_group'][0] != 'all': - raise exception.ApiError('only group "all" is supported') + raise exception.ApiError(_('only group "all" is supported')) if not operation_type in ['add', 'remove']: - raise exception.ApiError('operation_type must be add or remove') + raise exception.ApiError(_('operation_type must be add or remove')) return self.image_service.modify(context, image_id, operation_type) def update_image(self, context, image_id, **kwargs): diff --git a/nova/api/ec2/metadatarequesthandler.py b/nova/api/ec2/metadatarequesthandler.py index 2f4f414cc..f832863a9 100644 --- a/nova/api/ec2/metadatarequesthandler.py +++ b/nova/api/ec2/metadatarequesthandler.py @@ -23,9 +23,13 @@ import logging import webob.dec import webob.exc +from nova import flags from nova.api.ec2 import cloud +FLAGS = flags.FLAGS + + class MetadataRequestHandler(object): """Serve metadata from the EC2 API.""" @@ -63,10 +67,13 @@ class MetadataRequestHandler(object): @webob.dec.wsgify def __call__(self, req): cc = cloud.CloudController() - meta_data = cc.get_metadata(req.remote_addr) + remote_address = req.remote_addr + if FLAGS.use_forwarded_for: + remote_address = req.headers.get('X-Forwarded-For', remote_address) + meta_data = cc.get_metadata(remote_address) if meta_data is None: - logging.error('Failed to get metadata for ip: %s' % - req.remote_addr) + logging.error(_('Failed to get metadata for ip: %s') % + remote_address) raise webob.exc.HTTPNotFound() data = self.lookup(req.path_info, meta_data) if data is None: diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 4ca108c4e..c49399f28 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -30,6 +30,7 @@ import webob.dec import webob.exc import webob +from nova import context from nova import flags from nova import utils from nova import wsgi @@ -44,16 +45,27 @@ from nova.auth import manager FLAGS = flags.FLAGS -flags.DEFINE_string('nova_api_auth', - 'nova.api.openstack.auth.BasicApiAuthManager', +flags.DEFINE_string('os_api_auth', + 'nova.api.openstack.auth.AuthMiddleware', 'The auth mechanism to use for the OpenStack API implemenation') +flags.DEFINE_string('os_api_ratelimiting', + 'nova.api.openstack.ratelimiting.RateLimitingMiddleware', + 'Default ratelimiting implementation for the Openstack API') + +flags.DEFINE_bool('allow_admin_api', + False, + 'When True, this API service will accept admin operations.') + class API(wsgi.Middleware): """WSGI entry point for all OpenStack API requests.""" def __init__(self): - app = AuthMiddleware(RateLimitingMiddleware(APIRouter())) + auth_middleware = utils.import_class(FLAGS.os_api_auth) + ratelimiting_middleware = \ + utils.import_class(FLAGS.os_api_ratelimiting) + app = auth_middleware(ratelimiting_middleware(APIRouter())) super(API, self).__init__(app) @webob.dec.wsgify @@ -61,104 +73,12 @@ class API(wsgi.Middleware): try: return req.get_response(self.application) except Exception as ex: - logging.warn("Caught error: %s" % str(ex)) - logging.debug(traceback.format_exc()) + logging.warn(_("Caught error: %s") % str(ex)) + logging.error(traceback.format_exc()) exc = webob.exc.HTTPInternalServerError(explanation=str(ex)) return faults.Fault(exc) -class AuthMiddleware(wsgi.Middleware): - """Authorize the openstack API request or return an HTTP Forbidden.""" - - def __init__(self, application): - self.auth_driver = utils.import_class(FLAGS.nova_api_auth)() - super(AuthMiddleware, self).__init__(application) - - @webob.dec.wsgify - def __call__(self, req): - if 'X-Auth-Token' not in req.headers: - return self.auth_driver.authenticate(req) - - user = self.auth_driver.authorize_token(req.headers["X-Auth-Token"]) - - if not user: - return faults.Fault(webob.exc.HTTPUnauthorized()) - - if 'nova.context' not in req.environ: - req.environ['nova.context'] = {} - req.environ['nova.context']['user'] = user - return self.application - - -class RateLimitingMiddleware(wsgi.Middleware): - """Rate limit incoming requests according to the OpenStack rate limits.""" - - def __init__(self, application, service_host=None): - """Create a rate limiting middleware that wraps the given application. - - By default, rate counters are stored in memory. If service_host is - specified, the middleware instead relies on the ratelimiting.WSGIApp - at the given host+port to keep rate counters. - """ - super(RateLimitingMiddleware, self).__init__(application) - if not service_host: - #TODO(gundlach): These limits were based on limitations of Cloud - #Servers. We should revisit them in Nova. - self.limiter = ratelimiting.Limiter(limits={ - 'DELETE': (100, ratelimiting.PER_MINUTE), - 'PUT': (10, ratelimiting.PER_MINUTE), - 'POST': (10, ratelimiting.PER_MINUTE), - 'POST servers': (50, ratelimiting.PER_DAY), - 'GET changes-since': (3, ratelimiting.PER_MINUTE), - }) - else: - self.limiter = ratelimiting.WSGIAppProxy(service_host) - - @webob.dec.wsgify - def __call__(self, req): - """Rate limit the request. - - If the request should be rate limited, return a 413 status with a - Retry-After header giving the time when the request would succeed. - """ - user_id = req.environ['nova.context']['user']['id'] - action_name = self.get_action_name(req) - if not action_name: - # Not rate limited - return self.application - delay = self.get_delay(action_name, user_id) - if delay: - # TODO(gundlach): Get the retry-after format correct. - exc = webob.exc.HTTPRequestEntityTooLarge( - explanation='Too many requests.', - headers={'Retry-After': time.time() + delay}) - raise faults.Fault(exc) - return self.application - - def get_delay(self, action_name, username): - """Return the delay for the given action and username, or None if - the action would not be rate limited. - """ - if action_name == 'POST servers': - # "POST servers" is a POST, so it counts against "POST" too. - # Attempt the "POST" first, lest we are rate limited by "POST" but - # use up a precious "POST servers" call. - delay = self.limiter.perform("POST", username=username) - if delay: - return delay - return self.limiter.perform(action_name, username=username) - - def get_action_name(self, req): - """Return the action name for this request.""" - if req.method == 'GET' and 'changes-since' in req.GET: - return 'GET changes-since' - if req.method == 'POST' and req.path_info.startswith('/servers'): - return 'POST servers' - if req.method in ['PUT', 'POST', 'DELETE']: - return req.method - return None - - class APIRouter(wsgi.Router): """ Routes requests on the OpenStack API to the appropriate controller @@ -167,9 +87,16 @@ class APIRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() + + server_members = {'action': 'POST'} + if FLAGS.allow_admin_api: + logging.debug("Including admin operations in API.") + server_members['pause'] = 'POST' + server_members['unpause'] = 'POST' + mapper.resource("server", "servers", controller=servers.Controller(), collection={'detail': 'GET'}, - member={'action': 'POST'}) + member=server_members) mapper.resource("backup_schedule", "backup_schedules", controller=backup_schedules.Controller(), @@ -184,22 +111,3 @@ class APIRouter(wsgi.Router): controller=sharedipgroups.Controller()) super(APIRouter, self).__init__(mapper) - - -def limited(items, req): - """Return a slice of items according to requested offset and limit. - - items - a sliceable - req - wobob.Request possibly containing offset and limit GET variables. - offset is where to start in the list, and limit is the maximum number - of items to return. - - If limit is not specified, 0, or > 1000, defaults to 1000. - """ - offset = int(req.GET.get('offset', 0)) - limit = int(req.GET.get('limit', 0)) - if not limit: - limit = 1000 - limit = min(1000, limit) - range_end = offset + limit - return items[offset:range_end] diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 205035915..e24e58fd3 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -1,3 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License.import datetime + import datetime import hashlib import json @@ -7,29 +24,45 @@ import webob.exc import webob.dec from nova import auth +from nova import context from nova import db from nova import flags from nova import manager from nova import utils +from nova import wsgi from nova.api.openstack import faults FLAGS = flags.FLAGS -class Context(object): - pass - - -class BasicApiAuthManager(object): - """ Implements a somewhat rudimentary version of OpenStack Auth""" +class AuthMiddleware(wsgi.Middleware): + """Authorize the openstack API request or return an HTTP Forbidden.""" - def __init__(self, db_driver=None): + def __init__(self, application, db_driver=None): if not db_driver: db_driver = FLAGS.db_driver self.db = utils.import_object(db_driver) self.auth = auth.manager.AuthManager() - self.context = Context() - super(BasicApiAuthManager, self).__init__() + super(AuthMiddleware, self).__init__(application) + + @webob.dec.wsgify + def __call__(self, req): + if not self.has_authentication(req): + return self.authenticate(req) + + user = self.get_user_by_authentication(req) + + if not user: + return faults.Fault(webob.exc.HTTPUnauthorized()) + + req.environ['nova.context'] = context.RequestContext(user, user) + return self.application + + def has_authentication(self, req): + return 'X-Auth-Token' in req.headers + + def get_user_by_authentication(self, req): + return self.authorize_token(req.headers["X-Auth-Token"]) def authenticate(self, req): # Unless the request is explicitly made against /<version>/ don't @@ -68,15 +101,14 @@ class BasicApiAuthManager(object): This method will also remove the token if the timestamp is older than 2 days ago. """ - token = self.db.auth_get_token(self.context, token_hash) + ctxt = context.get_admin_context() + token = self.db.auth_get_token(ctxt, token_hash) if token: delta = datetime.datetime.now() - token.created_at if delta.days >= 2: - self.db.auth_destroy_token(self.context, token) + self.db.auth_destroy_token(ctxt, token) else: - #TODO(gundlach): Why not just return dict(id=token.user_id)? - user = self.auth.get_user(token.user_id) - return {'id': user.id} + return self.auth.get_user(token.user_id) return None def _authorize_user(self, username, key, req): @@ -86,6 +118,7 @@ class BasicApiAuthManager(object): key - string API key req - webob.Request object """ + ctxt = context.get_admin_context() user = self.auth.get_user_from_access_key(key) if user and user.name == username: token_hash = hashlib.sha1('%s%s%f' % (username, key, @@ -97,6 +130,6 @@ class BasicApiAuthManager(object): token_dict['server_management_url'] = req.url token_dict['storage_url'] = '' token_dict['user_id'] = user.id - token = self.db.auth_create_token(self.context, token_dict) + token = self.db.auth_create_token(ctxt, token_dict) return token, user return None, None diff --git a/nova/api/openstack/backup_schedules.py b/nova/api/openstack/backup_schedules.py index 3ed691d7b..fc70b5c6c 100644 --- a/nova/api/openstack/backup_schedules.py +++ b/nova/api/openstack/backup_schedules.py @@ -24,6 +24,7 @@ import nova.image.service class Controller(wsgi.Controller): + def __init__(self): pass diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py new file mode 100644 index 000000000..ac0572c96 --- /dev/null +++ b/nova/api/openstack/common.py @@ -0,0 +1,36 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +def limited(items, req): + """Return a slice of items according to requested offset and limit. + + items - a sliceable + req - wobob.Request possibly containing offset and limit GET variables. + offset is where to start in the list, and limit is the maximum number + of items to return. + + If limit is not specified, 0, or > 1000, defaults to 1000. + """ + + offset = int(req.GET.get('offset', 0)) + limit = int(req.GET.get('limit', 0)) + if not limit: + limit = 1000 + limit = min(1000, limit) + range_end = offset + limit + return items[offset:range_end] diff --git a/nova/api/openstack/flavors.py b/nova/api/openstack/flavors.py index f23f74fd1..f620d4107 100644 --- a/nova/api/openstack/flavors.py +++ b/nova/api/openstack/flavors.py @@ -18,6 +18,7 @@ from webob import exc from nova.api.openstack import faults +from nova.api.openstack import common from nova.compute import instance_types from nova import wsgi import nova.api.openstack @@ -39,7 +40,7 @@ class Controller(wsgi.Controller): def detail(self, req): """Return all flavors in detail.""" items = [self.show(req, id)['flavor'] for id in self._all_ids()] - items = nova.api.openstack.limited(items, req) + items = common.limited(items, req) return dict(flavors=items) def show(self, req, id): diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index cdbdc9bdd..d3312aba8 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -17,12 +17,13 @@ from webob import exc -from nova import context from nova import flags from nova import utils from nova import wsgi import nova.api.openstack import nova.image.service + +from nova.api.openstack import common from nova.api.openstack import faults @@ -47,23 +48,20 @@ class Controller(wsgi.Controller): def detail(self, req): """Return all public images in detail.""" - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) try: - images = self._service.detail(ctxt) - images = nova.api.openstack.limited(images, req) + images = self._service.detail(req.environ['nova.context']) + images = common.limited(images, req) except NotImplementedError: # Emulate detail() using repeated calls to show() + ctxt = req.environ['nova.context'] images = self._service.index(ctxt) - images = nova.api.openstack.limited(images, req) + images = common.limited(images, req) images = [self._service.show(ctxt, i['id']) for i in images] return dict(images=images) def show(self, req, id): """Return data about the given image id.""" - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) - return dict(image=self._service.show(ctxt, id)) + return dict(image=self._service.show(req.environ['nova.context'], id)) def delete(self, req, id): # Only public images are supported for now. diff --git a/nova/api/openstack/ratelimiting/__init__.py b/nova/api/openstack/ratelimiting/__init__.py index 918caf055..91a8b2e55 100644 --- a/nova/api/openstack/ratelimiting/__init__.py +++ b/nova/api/openstack/ratelimiting/__init__.py @@ -1,3 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License.import datetime + """Rate limiting of arbitrary actions.""" import httplib @@ -6,6 +23,8 @@ import urllib import webob.dec import webob.exc +from nova import wsgi +from nova.api.openstack import faults # Convenience constants for the limits dictionary passed to Limiter(). PER_SECOND = 1 @@ -14,6 +33,83 @@ PER_HOUR = 60 * 60 PER_DAY = 60 * 60 * 24 +class RateLimitingMiddleware(wsgi.Middleware): + """Rate limit incoming requests according to the OpenStack rate limits.""" + + def __init__(self, application, service_host=None): + """Create a rate limiting middleware that wraps the given application. + + By default, rate counters are stored in memory. If service_host is + specified, the middleware instead relies on the ratelimiting.WSGIApp + at the given host+port to keep rate counters. + """ + if not service_host: + #TODO(gundlach): These limits were based on limitations of Cloud + #Servers. We should revisit them in Nova. + self.limiter = Limiter(limits={ + 'DELETE': (100, PER_MINUTE), + 'PUT': (10, PER_MINUTE), + 'POST': (10, PER_MINUTE), + 'POST servers': (50, PER_DAY), + 'GET changes-since': (3, PER_MINUTE), + }) + else: + self.limiter = WSGIAppProxy(service_host) + super(RateLimitingMiddleware, self).__init__(application) + + @webob.dec.wsgify + def __call__(self, req): + """Rate limit the request. + + If the request should be rate limited, return a 413 status with a + Retry-After header giving the time when the request would succeed. + """ + return self.limited_request(req, self.application) + + def limited_request(self, req, application): + """Rate limit the request. + + If the request should be rate limited, return a 413 status with a + Retry-After header giving the time when the request would succeed. + """ + action_name = self.get_action_name(req) + if not action_name: + # Not rate limited + return application + delay = self.get_delay(action_name, + req.environ['nova.context'].user_id) + if delay: + # TODO(gundlach): Get the retry-after format correct. + exc = webob.exc.HTTPRequestEntityTooLarge( + explanation=('Too many requests.'), + headers={'Retry-After': time.time() + delay}) + raise faults.Fault(exc) + return application + + def get_delay(self, action_name, username): + """Return the delay for the given action and username, or None if + the action would not be rate limited. + """ + if action_name == 'POST servers': + # "POST servers" is a POST, so it counts against "POST" too. + # Attempt the "POST" first, lest we are rate limited by "POST" but + # use up a precious "POST servers" call. + delay = self.limiter.perform("POST", username=username) + if delay: + return delay + return self.limiter.perform(action_name, username=username) + + def get_action_name(self, req): + """Return the action name for this request.""" + if req.method == 'GET' and 'changes-since' in req.GET: + return 'GET changes-since' + if req.method == 'POST' and req.path_info.startswith('/servers'): + return 'POST servers' + if req.method in ['PUT', 'POST', 'DELETE']: + return req.method + return None + + class Limiter(object): """Class providing rate limiting of arbitrary actions.""" diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index e7f765c02..8d60e2cab 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -15,23 +15,24 @@ # License for the specific language governing permissions and limitations # under the License. -import webob +import logging +import traceback + from webob import exc -from nova import flags -from nova import rpc -from nova import utils +from nova import exception from nova import wsgi -from nova import context -from nova.api import cloud +from nova.api.openstack import common from nova.api.openstack import faults +from nova.auth import manager as auth_manager from nova.compute import api as compute_api from nova.compute import instance_types from nova.compute import power_state import nova.api.openstack -import nova.image.service -FLAGS = flags.FLAGS + +LOG = logging.getLogger('server') +LOG.setLevel(logging.DEBUG) def _entity_list(entities): @@ -79,11 +80,7 @@ class Controller(wsgi.Controller): "server": ["id", "imageId", "name", "flavorId", "hostId", "status", "progress"]}}} - def __init__(self, db_driver=None): - if not db_driver: - db_driver = FLAGS.db_driver - self.db_driver = utils.import_object(db_driver) - self.network_manager = utils.import_object(FLAGS.network_manager) + def __init__(self): self.compute_api = compute_api.ComputeAPI() super(Controller, self).__init__() @@ -100,32 +97,29 @@ class Controller(wsgi.Controller): entity_maker - either _entity_detail or _entity_inst """ - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) - instance_list = self.db_driver.instance_get_all_by_user(ctxt, user_id) - limited_list = nova.api.openstack.limited(instance_list, req) + instance_list = self.compute_api.get_instances( + req.environ['nova.context']) + limited_list = common.limited(instance_list, req) res = [entity_maker(inst)['server'] for inst in limited_list] return _entity_list(res) def show(self, req, id): """ Returns server details by server id """ - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) - inst = self.db_driver.instance_get_by_internal_id(ctxt, int(id)) - if inst: - if inst.user_id == user_id: - return _entity_detail(inst) - raise faults.Fault(exc.HTTPNotFound()) + try: + instance = self.compute_api.get_instance( + req.environ['nova.context'], int(id)) + return _entity_detail(instance) + except exception.NotFound: + return faults.Fault(exc.HTTPNotFound()) def delete(self, req, id): """ Destroys a server """ - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) - instance = self.db_driver.instance_get_by_internal_id(ctxt, int(id)) - if instance and instance['user_id'] == user_id: - self.db_driver.instance_destroy(ctxt, id) - return faults.Fault(exc.HTTPAccepted()) - return faults.Fault(exc.HTTPNotFound()) + try: + self.compute_api.delete_instance(req.environ['nova.context'], + int(id)) + except exception.NotFound: + return faults.Fault(exc.HTTPNotFound()) + return exc.HTTPAccepted() def create(self, req): """ Creates a new server for a given user """ @@ -133,15 +127,13 @@ class Controller(wsgi.Controller): if not env: return faults.Fault(exc.HTTPUnprocessableEntity()) - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) - key_pair = self.db_driver.key_pair_get_all_by_user(None, user_id)[0] - instances = self.compute_api.create_instances(ctxt, + key_pair = auth_manager.AuthManager.get_key_pairs( + req.environ['nova.context'])[0] + instances = self.compute_api.create_instances( + req.environ['nova.context'], instance_types.get_by_flavor_id(env['server']['flavorId']), - utils.import_object(FLAGS.image_service), env['server']['imageId'], - self._get_network_topic(ctxt), - name=env['server']['name'], + display_name=env['server']['name'], description=env['server']['name'], key_name=key_pair['name'], key_data=key_pair['public_key']) @@ -149,51 +141,58 @@ class Controller(wsgi.Controller): def update(self, req, id): """ Updates the server name or password """ - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) - inst_dict = self._deserialize(req.body, req) - if not inst_dict: return faults.Fault(exc.HTTPUnprocessableEntity()) - instance = self.db_driver.instance_get_by_internal_id(ctxt, int(id)) - if not instance or instance.user_id != user_id: - return faults.Fault(exc.HTTPNotFound()) - update_dict = {} if 'adminPass' in inst_dict['server']: update_dict['admin_pass'] = inst_dict['server']['adminPass'] if 'name' in inst_dict['server']: update_dict['display_name'] = inst_dict['server']['name'] - self.compute_api.update_instance(ctxt, instance['id'], update_dict) + try: + self.compute_api.update_instance(req.environ['nova.context'], + instance['id'], + **update_dict) + except exception.NotFound: + return faults.Fault(exc.HTTPNotFound()) return exc.HTTPNoContent() def action(self, req, id): - """ multi-purpose method used to reboot, rebuild, and + """ Multi-purpose method used to reboot, rebuild, and resize a server """ - user_id = req.environ['nova.context']['user']['id'] - ctxt = context.RequestContext(user_id, user_id) input_dict = self._deserialize(req.body, req) try: reboot_type = input_dict['reboot']['type'] except Exception: - raise faults.Fault(webob.exc.HTTPNotImplemented()) - inst_ref = self.db.instance_get_by_internal_id(ctxt, int(id)) - if not inst_ref or (inst_ref and not inst_ref.user_id == user_id): + raise faults.Fault(exc.HTTPNotImplemented()) + try: + # TODO(gundlach): pass reboot_type, support soft reboot in + # virt driver + self.compute_api.reboot(req.environ['nova.context'], id) + except: + return faults.Fault(exc.HTTPUnprocessableEntity()) + return exc.HTTPAccepted() + + def pause(self, req, id): + """ Permit Admins to Pause the server. """ + ctxt = req.environ['nova.context'] + try: + self.compute_api.pause(ctxt, id) + except: + readable = traceback.format_exc() + logging.error("Compute.api::pause %s", readable) + return faults.Fault(exc.HTTPUnprocessableEntity()) + return exc.HTTPAccepted() + + def unpause(self, req, id): + """ Permit Admins to Unpause the server. """ + ctxt = req.environ['nova.context'] + try: + self.compute_api.unpause(ctxt, id) + except: + readable = traceback.format_exc() + logging.error("Compute.api::unpause %s", readable) return faults.Fault(exc.HTTPUnprocessableEntity()) - #TODO(gundlach): pass reboot_type, support soft reboot in - #virt driver - cloud.reboot(id) - - def _get_network_topic(self, context): - """Retrieves the network host for a project""" - network_ref = self.network_manager.get_network(context) - host = network_ref['host'] - if not host: - host = rpc.call(context, - FLAGS.network_topic, - {"method": "set_network_host", - "args": {"network_id": network_ref['id']}}) - return self.db_driver.queue_get_for(context, FLAGS.network_topic, host) + return exc.HTTPAccepted() diff --git a/nova/api/openstack/sharedipgroups.py b/nova/api/openstack/sharedipgroups.py index e805ca9f7..75d02905c 100644 --- a/nova/api/openstack/sharedipgroups.py +++ b/nova/api/openstack/sharedipgroups.py @@ -19,4 +19,22 @@ from nova import wsgi class Controller(wsgi.Controller): - pass + """ The Shared IP Groups Controller for the Openstack API """ + + def index(self, req): + raise NotImplementedError + + def show(self, req, id): + raise NotImplementedError + + def update(self, req, id): + raise NotImplementedError + + def delete(self, req, id): + raise NotImplementedError + + def detail(self, req): + raise NotImplementedError + + def create(self, req): + raise NotImplementedError diff --git a/nova/auth/dbdriver.py b/nova/auth/dbdriver.py index a1584322b..47e435cb6 100644 --- a/nova/auth/dbdriver.py +++ b/nova/auth/dbdriver.py @@ -37,7 +37,6 @@ class DbDriver(object): def __init__(self): """Imports the LDAP module""" pass - db def __enter__(self): return self @@ -83,7 +82,7 @@ class DbDriver(object): user_ref = db.user_create(context.get_admin_context(), values) return self._db_user_to_auth_user(user_ref) except exception.Duplicate, e: - raise exception.Duplicate('User %s already exists' % name) + raise exception.Duplicate(_('User %s already exists') % name) def _db_user_to_auth_user(self, user_ref): return {'id': user_ref['id'], @@ -105,8 +104,9 @@ class DbDriver(object): """Create a project""" manager = db.user_get(context.get_admin_context(), manager_uid) if not manager: - raise exception.NotFound("Project can't be created because " - "manager %s doesn't exist" % manager_uid) + raise exception.NotFound(_("Project can't be created because " + "manager %s doesn't exist") + % manager_uid) # description is a required attribute if description is None: @@ -133,8 +133,8 @@ class DbDriver(object): try: project = db.project_create(context.get_admin_context(), values) except exception.Duplicate: - raise exception.Duplicate("Project can't be created because " - "project %s already exists" % name) + raise exception.Duplicate(_("Project can't be created because " + "project %s already exists") % name) for member in members: db.project_add_member(context.get_admin_context(), @@ -155,8 +155,8 @@ class DbDriver(object): if manager_uid: manager = db.user_get(context.get_admin_context(), manager_uid) if not manager: - raise exception.NotFound("Project can't be modified because " - "manager %s doesn't exist" % + raise exception.NotFound(_("Project can't be modified because " + "manager %s doesn't exist") % manager_uid) values['project_manager'] = manager['id'] if description: @@ -243,8 +243,8 @@ class DbDriver(object): def _validate_user_and_project(self, user_id, project_id): user = db.user_get(context.get_admin_context(), user_id) if not user: - raise exception.NotFound('User "%s" not found' % user_id) + raise exception.NotFound(_('User "%s" not found') % user_id) project = db.project_get(context.get_admin_context(), project_id) if not project: - raise exception.NotFound('Project "%s" not found' % project_id) + raise exception.NotFound(_('Project "%s" not found') % project_id) return user, project diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index 46e0135b4..4466051f0 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -15,7 +15,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -"""Fake LDAP server for test harness, backs to ReDIS. +"""Fake LDAP server for test harness. This class does very little error checking, and knows nothing about ldap class definitions. It implements the minimum emulation of the python ldap @@ -23,34 +23,65 @@ library to work with nova. """ +import fnmatch import json -import redis -from nova import flags -FLAGS = flags.FLAGS -flags.DEFINE_string('redis_host', '127.0.0.1', - 'Host that redis is running on.') -flags.DEFINE_integer('redis_port', 6379, - 'Port that redis is running on.') -flags.DEFINE_integer('redis_db', 0, 'Multiple DB keeps tests away') - - -class Redis(object): +class Store(object): def __init__(self): if hasattr(self.__class__, '_instance'): - raise Exception('Attempted to instantiate singleton') + raise Exception(_('Attempted to instantiate singleton')) @classmethod def instance(cls): if not hasattr(cls, '_instance'): - inst = redis.Redis(host=FLAGS.redis_host, - port=FLAGS.redis_port, - db=FLAGS.redis_db) - cls._instance = inst + cls._instance = _StorageDict() return cls._instance +class _StorageDict(dict): + def keys(self, pat=None): + ret = super(_StorageDict, self).keys() + if pat is not None: + ret = fnmatch.filter(ret, pat) + return ret + + def delete(self, key): + try: + del self[key] + except KeyError: + pass + + def flushdb(self): + self.clear() + + def hgetall(self, key): + """Returns the hash for the given key; creates + the hash if the key doesn't exist.""" + try: + return self[key] + except KeyError: + self[key] = {} + return self[key] + + def hget(self, key, field): + hashdict = self.hgetall(key) + try: + return hashdict[field] + except KeyError: + hashdict[field] = {} + return hashdict[field] + + def hset(self, key, field, val): + hashdict = self.hgetall(key) + hashdict[field] = val + + def hmset(self, key, value_dict): + hashdict = self.hgetall(key) + for field, val in value_dict.items(): + hashdict[field] = val + + SCOPE_BASE = 0 SCOPE_ONELEVEL = 1 # Not implemented SCOPE_SUBTREE = 2 @@ -119,6 +150,9 @@ def _match(key, value, attrs): """Match a given key and value against an attribute list.""" if key not in attrs: return False + # This is a wild card search. Implemented as all or nothing for now. + if value == "*": + return True if key != "objectclass": return value in attrs[key] # it is an objectclass check, so check subclasses @@ -169,8 +203,6 @@ def _to_json(unencoded): class FakeLDAP(object): - #TODO(vish): refactor this class to use a wrapper instead of accessing - # redis directly """Fake LDAP connection.""" def simple_bind_s(self, dn, password): @@ -183,14 +215,13 @@ class FakeLDAP(object): def add_s(self, dn, attr): """Add an object with the specified attributes at dn.""" - key = "%s%s" % (self.__redis_prefix, dn) - + key = "%s%s" % (self.__prefix, dn) value_dict = dict([(k, _to_json(v)) for k, v in attr]) - Redis.instance().hmset(key, value_dict) + Store.instance().hmset(key, value_dict) def delete_s(self, dn): """Remove the ldap object at specified dn.""" - Redis.instance().delete("%s%s" % (self.__redis_prefix, dn)) + Store.instance().delete("%s%s" % (self.__prefix, dn)) def modify_s(self, dn, attrs): """Modify the object at dn using the attribute list. @@ -201,18 +232,18 @@ class FakeLDAP(object): ([MOD_ADD | MOD_DELETE | MOD_REPACE], attribute, value) """ - redis = Redis.instance() - key = "%s%s" % (self.__redis_prefix, dn) + store = Store.instance() + key = "%s%s" % (self.__prefix, dn) for cmd, k, v in attrs: - values = _from_json(redis.hget(key, k)) + values = _from_json(store.hget(key, k)) if cmd == MOD_ADD: values.append(v) elif cmd == MOD_REPLACE: values = [v] else: values.remove(v) - values = redis.hset(key, k, _to_json(values)) + values = store.hset(key, k, _to_json(values)) def search_s(self, dn, scope, query=None, fields=None): """Search for all matching objects under dn using the query. @@ -226,16 +257,17 @@ class FakeLDAP(object): """ if scope != SCOPE_BASE and scope != SCOPE_SUBTREE: raise NotImplementedError(str(scope)) - redis = Redis.instance() + store = Store.instance() if scope == SCOPE_BASE: - keys = ["%s%s" % (self.__redis_prefix, dn)] + keys = ["%s%s" % (self.__prefix, dn)] else: - keys = redis.keys("%s*%s" % (self.__redis_prefix, dn)) + keys = store.keys("%s*%s" % (self.__prefix, dn)) + objects = [] for key in keys: - # get the attributes from redis - attrs = redis.hgetall(key) - # turn the values from redis into lists + # get the attributes from the store + attrs = store.hgetall(key) + # turn the values from the store into lists # pylint: disable-msg=E1103 attrs = dict([(k, _from_json(v)) for k, v in attrs.iteritems()]) @@ -244,13 +276,13 @@ class FakeLDAP(object): # filter the attributes by fields attrs = dict([(k, v) for k, v in attrs.iteritems() if not fields or k in fields]) - objects.append((key[len(self.__redis_prefix):], attrs)) + objects.append((key[len(self.__prefix):], attrs)) # pylint: enable-msg=E1103 if objects == []: raise NO_SUCH_OBJECT() return objects @property - def __redis_prefix(self): # pylint: disable-msg=R0201 - """Get the prefix to use for all redis keys.""" + def __prefix(self): # pylint: disable-msg=R0201 + """Get the prefix to use for all keys.""" return 'ldap:' diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index ceade1d65..7616ff112 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -32,14 +32,21 @@ from nova import flags FLAGS = flags.FLAGS +flags.DEFINE_integer('ldap_schema_version', 2, + 'Current version of the LDAP schema') flags.DEFINE_string('ldap_url', 'ldap://localhost', 'Point this at your ldap server') flags.DEFINE_string('ldap_password', 'changeme', 'LDAP password') flags.DEFINE_string('ldap_user_dn', 'cn=Manager,dc=example,dc=com', 'DN of admin user') +flags.DEFINE_string('ldap_user_id_attribute', 'uid', 'Attribute to use as id') +flags.DEFINE_string('ldap_user_name_attribute', 'cn', + 'Attribute to use as name') flags.DEFINE_string('ldap_user_unit', 'Users', 'OID for Users') flags.DEFINE_string('ldap_user_subtree', 'ou=Users,dc=example,dc=com', 'OU for Users') +flags.DEFINE_boolean('ldap_user_modify_only', False, + 'Modify attributes for users instead of creating/deleting') flags.DEFINE_string('ldap_project_subtree', 'ou=Groups,dc=example,dc=com', 'OU for Projects') flags.DEFINE_string('role_project_subtree', 'ou=Groups,dc=example,dc=com', @@ -71,10 +78,20 @@ class LdapDriver(object): Defines enter and exit and therefore supports the with/as syntax. """ + project_pattern = '(owner=*)' + isadmin_attribute = 'isNovaAdmin' + project_attribute = 'owner' + project_objectclass = 'groupOfNames' + def __init__(self): """Imports the LDAP module""" self.ldap = __import__('ldap') self.conn = None + if FLAGS.ldap_schema_version == 1: + LdapDriver.project_pattern = '(objectclass=novaProject)' + LdapDriver.isadmin_attribute = 'isAdmin' + LdapDriver.project_attribute = 'projectManager' + LdapDriver.project_objectclass = 'novaProject' def __enter__(self): """Creates the connection to LDAP""" @@ -89,8 +106,7 @@ class LdapDriver(object): def get_user(self, uid): """Retrieve user by id""" - attr = self.__find_object(self.__uid_to_dn(uid), - '(objectclass=novaUser)') + attr = self.__get_ldap_user(uid) return self.__to_user(attr) def get_user_from_access_key(self, access): @@ -103,18 +119,23 @@ class LdapDriver(object): """Retrieve project by id""" dn = 'cn=%s,%s' % (pid, FLAGS.ldap_project_subtree) - attr = self.__find_object(dn, '(objectclass=novaProject)') + attr = self.__find_object(dn, LdapDriver.project_pattern) return self.__to_project(attr) def get_users(self): """Retrieve list of users""" attrs = self.__find_objects(FLAGS.ldap_user_subtree, - '(objectclass=novaUser)') - return [self.__to_user(attr) for attr in attrs] + '(objectclass=novaUser)') + users = [] + for attr in attrs: + user = self.__to_user(attr) + if user is not None: + users.append(user) + return users def get_projects(self, uid=None): """Retrieve list of projects""" - pattern = '(objectclass=novaProject)' + pattern = LdapDriver.project_pattern if uid: pattern = "(&%s(member=%s))" % (pattern, self.__uid_to_dn(uid)) attrs = self.__find_objects(FLAGS.ldap_project_subtree, @@ -125,51 +146,85 @@ class LdapDriver(object): """Create a user""" if self.__user_exists(name): raise exception.Duplicate("LDAP user %s already exists" % name) - attr = [ - ('objectclass', ['person', - 'organizationalPerson', - 'inetOrgPerson', - 'novaUser']), - ('ou', [FLAGS.ldap_user_unit]), - ('uid', [name]), - ('sn', [name]), - ('cn', [name]), - ('secretKey', [secret_key]), - ('accessKey', [access_key]), - ('isAdmin', [str(is_admin).upper()]), - ] - self.conn.add_s(self.__uid_to_dn(name), attr) - return self.__to_user(dict(attr)) + if FLAGS.ldap_user_modify_only: + if self.__ldap_user_exists(name): + # Retrieve user by name + user = self.__get_ldap_user(name) + # Entry could be malformed, test for missing attrs. + # Malformed entries are useless, replace attributes found. + attr = [] + if 'secretKey' in user.keys(): + attr.append((self.ldap.MOD_REPLACE, 'secretKey', + [secret_key])) + else: + attr.append((self.ldap.MOD_ADD, 'secretKey', + [secret_key])) + if 'accessKey' in user.keys(): + attr.append((self.ldap.MOD_REPLACE, 'accessKey', + [access_key])) + else: + attr.append((self.ldap.MOD_ADD, 'accessKey', + [access_key])) + if LdapDriver.isadmin_attribute in user.keys(): + attr.append((self.ldap.MOD_REPLACE, + LdapDriver.isadmin_attribute, + [str(is_admin).upper()])) + else: + attr.append((self.ldap.MOD_ADD, + LdapDriver.isadmin_attribute, + [str(is_admin).upper()])) + self.conn.modify_s(self.__uid_to_dn(name), attr) + return self.get_user(name) + else: + raise exception.NotFound(_("LDAP object for %s doesn't exist") + % name) + else: + attr = [ + ('objectclass', ['person', + 'organizationalPerson', + 'inetOrgPerson', + 'novaUser']), + ('ou', [FLAGS.ldap_user_unit]), + (FLAGS.ldap_user_id_attribute, [name]), + ('sn', [name]), + (FLAGS.ldap_user_name_attribute, [name]), + ('secretKey', [secret_key]), + ('accessKey', [access_key]), + (LdapDriver.isadmin_attribute, [str(is_admin).upper()]), + ] + self.conn.add_s(self.__uid_to_dn(name), attr) + return self.__to_user(dict(attr)) def create_project(self, name, manager_uid, description=None, member_uids=None): """Create a project""" if self.__project_exists(name): - raise exception.Duplicate("Project can't be created because " - "project %s already exists" % name) + raise exception.Duplicate(_("Project can't be created because " + "project %s already exists") % name) if not self.__user_exists(manager_uid): - raise exception.NotFound("Project can't be created because " - "manager %s doesn't exist" % manager_uid) + raise exception.NotFound(_("Project can't be created because " + "manager %s doesn't exist") + % manager_uid) manager_dn = self.__uid_to_dn(manager_uid) # description is a required attribute if description is None: description = name members = [] - if member_uids != None: + if member_uids is not None: for member_uid in member_uids: if not self.__user_exists(member_uid): - raise exception.NotFound("Project can't be created " - "because user %s doesn't exist" + raise exception.NotFound(_("Project can't be created " + "because user %s doesn't exist") % member_uid) members.append(self.__uid_to_dn(member_uid)) # always add the manager as a member because members is required if not manager_dn in members: members.append(manager_dn) attr = [ - ('objectclass', ['novaProject']), + ('objectclass', [LdapDriver.project_objectclass]), ('cn', [name]), ('description', [description]), - ('projectManager', [manager_dn]), + (LdapDriver.project_attribute, [manager_dn]), ('member', members)] self.conn.add_s('cn=%s,%s' % (name, FLAGS.ldap_project_subtree), attr) return self.__to_project(dict(attr)) @@ -181,11 +236,12 @@ class LdapDriver(object): attr = [] if manager_uid: if not self.__user_exists(manager_uid): - raise exception.NotFound("Project can't be modified because " - "manager %s doesn't exist" % - manager_uid) + raise exception.NotFound(_("Project can't be modified because " + "manager %s doesn't exist") + % manager_uid) manager_dn = self.__uid_to_dn(manager_uid) - attr.append((self.ldap.MOD_REPLACE, 'projectManager', manager_dn)) + attr.append((self.ldap.MOD_REPLACE, LdapDriver.project_attribute, + manager_dn)) if description: attr.append((self.ldap.MOD_REPLACE, 'description', description)) self.conn.modify_s('cn=%s,%s' % (project_id, @@ -245,10 +301,9 @@ class LdapDriver(object): return roles else: project_dn = 'cn=%s,%s' % (project_id, FLAGS.ldap_project_subtree) - roles = self.__find_objects(project_dn, - '(&(&(objectclass=groupOfNames)' - '(!(objectclass=novaProject)))' - '(member=%s))' % self.__uid_to_dn(uid)) + query = ('(&(&(objectclass=groupOfNames)(!%s))(member=%s))' % + (LdapDriver.project_pattern, self.__uid_to_dn(uid))) + roles = self.__find_objects(project_dn, query) return [role['cn'][0] for role in roles] def delete_user(self, uid): @@ -256,7 +311,25 @@ class LdapDriver(object): if not self.__user_exists(uid): raise exception.NotFound("User %s doesn't exist" % uid) self.__remove_from_all(uid) - self.conn.delete_s(self.__uid_to_dn(uid)) + if FLAGS.ldap_user_modify_only: + # Delete attributes + attr = [] + # Retrieve user by name + user = self.__get_ldap_user(uid) + if 'secretKey' in user.keys(): + attr.append((self.ldap.MOD_DELETE, 'secretKey', + user['secretKey'])) + if 'accessKey' in user.keys(): + attr.append((self.ldap.MOD_DELETE, 'accessKey', + user['accessKey'])) + if LdapDriver.isadmin_attribute in user.keys(): + attr.append((self.ldap.MOD_DELETE, + LdapDriver.isadmin_attribute, + user[LdapDriver.isadmin_attribute])) + self.conn.modify_s(self.__uid_to_dn(uid), attr) + else: + # Delete entry + self.conn.delete_s(self.__uid_to_dn(uid)) def delete_project(self, project_id): """Delete a project""" @@ -265,7 +338,7 @@ class LdapDriver(object): self.__delete_group(project_dn) def modify_user(self, uid, access_key=None, secret_key=None, admin=None): - """Modify an existing project""" + """Modify an existing user""" if not access_key and not secret_key and admin is None: return attr = [] @@ -274,16 +347,27 @@ class LdapDriver(object): if secret_key: attr.append((self.ldap.MOD_REPLACE, 'secretKey', secret_key)) if admin is not None: - attr.append((self.ldap.MOD_REPLACE, 'isAdmin', str(admin).upper())) + attr.append((self.ldap.MOD_REPLACE, LdapDriver.isadmin_attribute, + str(admin).upper())) self.conn.modify_s(self.__uid_to_dn(uid), attr) def __user_exists(self, uid): """Check if user exists""" - return self.get_user(uid) != None + return self.get_user(uid) is not None + + def __ldap_user_exists(self, uid): + """Check if the user exists in ldap""" + return self.__get_ldap_user(uid) is not None def __project_exists(self, project_id): """Check if project exists""" - return self.get_project(project_id) != None + return self.get_project(project_id) is not None + + def __get_ldap_user(self, uid): + """Retrieve LDAP user entry by id""" + attr = self.__find_object(self.__uid_to_dn(uid), + '(objectclass=novaUser)') + return attr def __find_object(self, dn, query=None, scope=None): """Find an object by dn and query""" @@ -318,24 +402,26 @@ class LdapDriver(object): def __find_role_dns(self, tree): """Find dns of role objects in given tree""" - return self.__find_dns(tree, - '(&(objectclass=groupOfNames)(!(objectclass=novaProject)))') + query = ('(&(objectclass=groupOfNames)(!%s))' % + LdapDriver.project_pattern) + return self.__find_dns(tree, query) def __find_group_dns_with_member(self, tree, uid): """Find dns of group objects in a given tree that contain member""" - dns = self.__find_dns(tree, - '(&(objectclass=groupOfNames)(member=%s))' % - self.__uid_to_dn(uid)) + query = ('(&(objectclass=groupOfNames)(member=%s))' % + self.__uid_to_dn(uid)) + dns = self.__find_dns(tree, query) return dns def __group_exists(self, dn): """Check if group exists""" - return self.__find_object(dn, '(objectclass=groupOfNames)') != None + query = '(objectclass=groupOfNames)' + return self.__find_object(dn, query) is not None @staticmethod def __role_to_dn(role, project_id=None): """Convert role to corresponding dn""" - if project_id == None: + if project_id is None: return FLAGS.__getitem__("ldap_%s" % role).value else: return 'cn=%s,cn=%s,%s' % (role, @@ -349,11 +435,12 @@ class LdapDriver(object): raise exception.Duplicate("Group can't be created because " "group %s already exists" % name) members = [] - if member_uids != None: + if member_uids is not None: for member_uid in member_uids: if not self.__user_exists(member_uid): raise exception.NotFound("Group can't be created " - "because user %s doesn't exist" % member_uid) + "because user %s doesn't exist" % + member_uid) members.append(self.__uid_to_dn(member_uid)) dn = self.__uid_to_dn(uid) if not dn in members: @@ -369,25 +456,25 @@ class LdapDriver(object): """Check if user is in group""" if not self.__user_exists(uid): raise exception.NotFound("User %s can't be searched in group " - "becuase the user doesn't exist" % (uid,)) + "because the user doesn't exist" % uid) if not self.__group_exists(group_dn): return False res = self.__find_object(group_dn, '(member=%s)' % self.__uid_to_dn(uid), self.ldap.SCOPE_BASE) - return res != None + return res is not None def __add_to_group(self, uid, group_dn): """Add user to group""" if not self.__user_exists(uid): raise exception.NotFound("User %s can't be added to the group " - "becuase the user doesn't exist" % (uid,)) + "because the user doesn't exist" % uid) if not self.__group_exists(group_dn): raise exception.NotFound("The group at dn %s doesn't exist" % - (group_dn,)) + group_dn) if self.__is_in_group(uid, group_dn): - raise exception.Duplicate("User %s is already a member of " - "the group %s" % (uid, group_dn)) + raise exception.Duplicate(_("User %s is already a member of " + "the group %s") % (uid, group_dn)) attr = [(self.ldap.MOD_ADD, 'member', self.__uid_to_dn(uid))] self.conn.modify_s(group_dn, attr) @@ -395,16 +482,16 @@ class LdapDriver(object): """Remove user from group""" if not self.__group_exists(group_dn): raise exception.NotFound("The group at dn %s doesn't exist" % - (group_dn,)) + group_dn) if not self.__user_exists(uid): raise exception.NotFound("User %s can't be removed from the " - "group because the user doesn't exist" % (uid,)) + "group because the user doesn't exist" % + uid) if not self.__is_in_group(uid, group_dn): raise exception.NotFound("User %s is not a member of the group" % - (uid,)) + uid) # NOTE(vish): remove user from group and any sub_groups - sub_dns = self.__find_group_dns_with_member( - group_dn, uid) + sub_dns = self.__find_group_dns_with_member(group_dn, uid) for sub_dn in sub_dns: self.__safe_remove_from_group(uid, sub_dn) @@ -415,15 +502,15 @@ class LdapDriver(object): try: self.conn.modify_s(group_dn, attr) except self.ldap.OBJECT_CLASS_VIOLATION: - logging.debug("Attempted to remove the last member of a group. " - "Deleting the group at %s instead.", group_dn) + logging.debug(_("Attempted to remove the last member of a group. " + "Deleting the group at %s instead."), group_dn) self.__delete_group(group_dn) def __remove_from_all(self, uid): """Remove user from all roles and projects""" if not self.__user_exists(uid): raise exception.NotFound("User %s can't be removed from all " - "because the user doesn't exist" % (uid,)) + "because the user doesn't exist" % uid) role_dns = self.__find_group_dns_with_member( FLAGS.role_project_subtree, uid) for role_dn in role_dns: @@ -436,7 +523,8 @@ class LdapDriver(object): def __delete_group(self, group_dn): """Delete Group""" if not self.__group_exists(group_dn): - raise exception.NotFound("Group at dn %s doesn't exist" % group_dn) + raise exception.NotFound(_("Group at dn %s doesn't exist") + % group_dn) self.conn.delete_s(group_dn) def __delete_roles(self, project_dn): @@ -447,24 +535,29 @@ class LdapDriver(object): @staticmethod def __to_user(attr): """Convert ldap attributes to User object""" - if attr == None: + if attr is None: + return None + if ('accessKey' in attr.keys() and 'secretKey' in attr.keys() \ + and LdapDriver.isadmin_attribute in attr.keys()): + return { + 'id': attr[FLAGS.ldap_user_id_attribute][0], + 'name': attr[FLAGS.ldap_user_name_attribute][0], + 'access': attr['accessKey'][0], + 'secret': attr['secretKey'][0], + 'admin': (attr[LdapDriver.isadmin_attribute][0] == 'TRUE')} + else: return None - return { - 'id': attr['uid'][0], - 'name': attr['cn'][0], - 'access': attr['accessKey'][0], - 'secret': attr['secretKey'][0], - 'admin': (attr['isAdmin'][0] == 'TRUE')} def __to_project(self, attr): """Convert ldap attributes to Project object""" - if attr == None: + if attr is None: return None member_dns = attr.get('member', []) return { 'id': attr['cn'][0], 'name': attr['cn'][0], - 'project_manager_id': self.__dn_to_uid(attr['projectManager'][0]), + 'project_manager_id': + self.__dn_to_uid(attr[LdapDriver.project_attribute][0]), 'description': attr.get('description', [None])[0], 'member_ids': [self.__dn_to_uid(x) for x in member_dns]} @@ -474,9 +567,10 @@ class LdapDriver(object): return dn.split(',')[0].split('=')[1] @staticmethod - def __uid_to_dn(dn): + def __uid_to_dn(uid): """Convert uid to dn""" - return 'uid=%s,%s' % (dn, FLAGS.ldap_user_subtree) + return (FLAGS.ldap_user_id_attribute + '=%s,%s' + % (uid, FLAGS.ldap_user_subtree)) class FakeLdapDriver(LdapDriver): diff --git a/nova/auth/manager.py b/nova/auth/manager.py index 7b2b68161..d3e266952 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -64,12 +64,9 @@ flags.DEFINE_string('credential_key_file', 'pk.pem', 'Filename of private key in credentials zip') flags.DEFINE_string('credential_cert_file', 'cert.pem', 'Filename of certificate in credentials zip') -flags.DEFINE_string('credential_rc_file', 'novarc', - 'Filename of rc in credentials zip') -flags.DEFINE_string('credential_cert_subject', - '/C=US/ST=California/L=MountainView/O=AnsoLabs/' - 'OU=NovaDev/CN=%s-%s', - 'Subject for certificate for users') +flags.DEFINE_string('credential_rc_file', '%src', + 'Filename of rc in credentials zip, %s will be ' + 'replaced by name of the region (nova by default)') flags.DEFINE_string('auth_driver', 'nova.auth.dbdriver.DbDriver', 'Driver that auth manager uses') @@ -257,12 +254,12 @@ class AuthManager(object): # TODO(vish): check for valid timestamp (access_key, _sep, project_id) = access.partition(':') - logging.info('Looking up user: %r', access_key) + logging.info(_('Looking up user: %r'), access_key) user = self.get_user_from_access_key(access_key) logging.info('user: %r', user) if user == None: - raise exception.NotFound('No user found for access key %s' % - access_key) + raise exception.NotFound(_('No user found for access key %s') + % access_key) # NOTE(vish): if we stop using project name as id we need better # logic to find a default project for user @@ -271,12 +268,12 @@ class AuthManager(object): project = self.get_project(project_id) if project == None: - raise exception.NotFound('No project called %s could be found' % - project_id) + raise exception.NotFound(_('No project called %s could be found') + % project_id) if not self.is_admin(user) and not self.is_project_member(user, project): - raise exception.NotFound('User %s is not a member of project %s' % - (user.id, project.id)) + raise exception.NotFound(_('User %s is not a member of project %s') + % (user.id, project.id)) if check_type == 's3': sign = signer.Signer(user.secret.encode()) expected_signature = sign.s3_authorization(headers, verb, path) @@ -284,7 +281,7 @@ class AuthManager(object): logging.debug('expected_signature: %s', expected_signature) logging.debug('signature: %s', signature) if signature != expected_signature: - raise exception.NotAuthorized('Signature does not match') + raise exception.NotAuthorized(_('Signature does not match')) elif check_type == 'ec2': # NOTE(vish): hmac can't handle unicode, so encode ensures that # secret isn't unicode @@ -294,7 +291,7 @@ class AuthManager(object): logging.debug('expected_signature: %s', expected_signature) logging.debug('signature: %s', signature) if signature != expected_signature: - raise exception.NotAuthorized('Signature does not match') + raise exception.NotAuthorized(_('Signature does not match')) return (user, project) def get_access_key(self, user, project): @@ -364,7 +361,7 @@ class AuthManager(object): with self.driver() as drv: if role == 'projectmanager': if not project: - raise exception.Error("Must specify project") + raise exception.Error(_("Must specify project")) return self.is_project_manager(user, project) global_role = drv.has_role(User.safe_id(user), @@ -398,9 +395,9 @@ class AuthManager(object): @param project: Project in which to add local role. """ if role not in FLAGS.allowed_roles: - raise exception.NotFound("The %s role can not be found" % role) + raise exception.NotFound(_("The %s role can not be found") % role) if project is not None and role in FLAGS.global_roles: - raise exception.NotFound("The %s role is global only" % role) + raise exception.NotFound(_("The %s role is global only") % role) with self.driver() as drv: drv.add_role(User.safe_id(user), role, Project.safe_id(project)) @@ -543,10 +540,10 @@ class AuthManager(object): """ network_ref = db.project_get_network(context.get_admin_context(), - Project.safe_id(project)) + Project.safe_id(project), False) - if not network_ref['vpn_public_port']: - raise exception.NotFound('project network data has not been set') + if not network_ref: + return (None, None) return (network_ref['vpn_public_address'], network_ref['vpn_public_port']) @@ -624,27 +621,41 @@ class AuthManager(object): with self.driver() as drv: drv.modify_user(uid, access_key, secret_key, admin) - def get_credentials(self, user, project=None): + @staticmethod + def get_key_pairs(context): + return db.key_pair_get_all_by_user(context.elevated(), context.user_id) + + def get_credentials(self, user, project=None, use_dmz=True): """Get credential zip for user in project""" if not isinstance(user, User): user = self.get_user(user) if project is None: project = user.id pid = Project.safe_id(project) - rc = self.__generate_rc(user.access, user.secret, pid) - private_key, signed_cert = self._generate_x509_cert(user.id, pid) + private_key, signed_cert = crypto.generate_x509_cert(user.id, pid) tmpdir = tempfile.mkdtemp() zf = os.path.join(tmpdir, "temp.zip") zippy = zipfile.ZipFile(zf, 'w') - zippy.writestr(FLAGS.credential_rc_file, rc) + if use_dmz and FLAGS.region_list: + regions = {} + for item in FLAGS.region_list: + region, _sep, region_host = item.partition("=") + regions[region] = region_host + else: + regions = {'nova': FLAGS.cc_host} + for region, host in regions.iteritems(): + rc = self.__generate_rc(user.access, + user.secret, + pid, + use_dmz, + host) + zippy.writestr(FLAGS.credential_rc_file % region, rc) + zippy.writestr(FLAGS.credential_key_file, private_key) zippy.writestr(FLAGS.credential_cert_file, signed_cert) - try: - (vpn_ip, vpn_port) = self.get_project_vpn_data(project) - except exception.NotFound: - vpn_ip = None + (vpn_ip, vpn_port) = self.get_project_vpn_data(project) if vpn_ip: configfile = open(FLAGS.vpn_client_template, "r") s = string.Template(configfile.read()) @@ -655,10 +666,9 @@ class AuthManager(object): port=vpn_port) zippy.writestr(FLAGS.credential_vpn_file, config) else: - logging.warn("No vpn data for project %s" % - pid) + logging.warn(_("No vpn data for project %s"), pid) - zippy.writestr(FLAGS.ca_file, crypto.fetch_ca(user.id)) + zippy.writestr(FLAGS.ca_file, crypto.fetch_ca(pid)) zippy.close() with open(zf, 'rb') as f: read_buffer = f.read() @@ -666,38 +676,38 @@ class AuthManager(object): shutil.rmtree(tmpdir) return read_buffer - def get_environment_rc(self, user, project=None): + def get_environment_rc(self, user, project=None, use_dmz=True): """Get credential zip for user in project""" if not isinstance(user, User): user = self.get_user(user) if project is None: project = user.id pid = Project.safe_id(project) - return self.__generate_rc(user.access, user.secret, pid) + return self.__generate_rc(user.access, user.secret, pid, use_dmz) @staticmethod - def __generate_rc(access, secret, pid): + def __generate_rc(access, secret, pid, use_dmz=True, host=None): """Generate rc file for user""" + if use_dmz: + cc_host = FLAGS.cc_dmz + else: + cc_host = FLAGS.cc_host + # NOTE(vish): Always use the dmz since it is used from inside the + # instance + s3_host = FLAGS.s3_dmz + if host: + s3_host = host + cc_host = host rc = open(FLAGS.credentials_template).read() rc = rc % {'access': access, 'project': pid, 'secret': secret, - 'ec2': FLAGS.ec2_url, - 's3': 'http://%s:%s' % (FLAGS.s3_host, FLAGS.s3_port), + 'ec2': '%s://%s:%s%s' % (FLAGS.ec2_prefix, + cc_host, + FLAGS.cc_port, + FLAGS.ec2_suffix), + 's3': 'http://%s:%s' % (s3_host, FLAGS.s3_port), 'nova': FLAGS.ca_file, 'cert': FLAGS.credential_cert_file, 'key': FLAGS.credential_key_file} return rc - - def _generate_x509_cert(self, uid, pid): - """Generate x509 cert for user""" - (private_key, csr) = crypto.generate_x509_cert( - self.__cert_subject(uid)) - # TODO(joshua): This should be async call back to the cloud controller - signed_cert = crypto.sign_csr(csr, pid) - return (private_key, signed_cert) - - @staticmethod - def __cert_subject(uid): - """Helper to generate cert subject""" - return FLAGS.credential_cert_subject % (uid, utils.isotime()) diff --git a/nova/auth/nova_openldap.schema b/nova/auth/nova_openldap.schema index 4047361de..539a5c42d 100644 --- a/nova/auth/nova_openldap.schema +++ b/nova/auth/nova_openldap.schema @@ -1,7 +1,9 @@ # # Person object for Nova # inetorgperson with extra attributes -# Author: Vishvananda Ishaya <vishvananda@yahoo.com> +# Schema version: 2 +# Authors: Vishvananda Ishaya <vishvananda@gmail.com> +# Ryan Lane <rlane@wikimedia.org> # # @@ -31,54 +33,18 @@ attributetype ( ) attributetype ( - novaAttrs:3 - NAME 'keyFingerprint' - DESC 'Fingerprint of private key' - EQUALITY caseIgnoreMatch - SUBSTR caseIgnoreSubstringsMatch - SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 - SINGLE-VALUE - ) - -attributetype ( novaAttrs:4 - NAME 'isAdmin' - DESC 'Is user an administrator?' + NAME 'isNovaAdmin' + DESC 'Is user an nova administrator?' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) -attributetype ( - novaAttrs:5 - NAME 'projectManager' - DESC 'Project Managers of a project' - SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 - ) - objectClass ( novaOCs:1 NAME 'novaUser' DESC 'access and secret keys' AUXILIARY - MUST ( uid ) - MAY ( accessKey $ secretKey $ isAdmin ) - ) - -objectClass ( - novaOCs:2 - NAME 'novaKeyPair' - DESC 'Key pair for User' - SUP top - STRUCTURAL - MUST ( cn $ sshPublicKey $ keyFingerprint ) - ) - -objectClass ( - novaOCs:3 - NAME 'novaProject' - DESC 'Container for project' - SUP groupOfNames - STRUCTURAL - MUST ( cn $ projectManager ) + MAY ( accessKey $ secretKey $ isNovaAdmin ) ) diff --git a/nova/auth/nova_sun.schema b/nova/auth/nova_sun.schema index e925e05e4..4a6a78839 100644 --- a/nova/auth/nova_sun.schema +++ b/nova/auth/nova_sun.schema @@ -1,16 +1,13 @@ # # Person object for Nova # inetorgperson with extra attributes -# Author: Vishvananda Ishaya <vishvananda@yahoo.com> -# Modified for strict RFC 4512 compatibility by: Ryan Lane <ryan@ryandlane.com> +# Schema version: 2 +# Authors: Vishvananda Ishaya <vishvananda@gmail.com> +# Ryan Lane <rlane@wikimedia.org> # # using internet experimental oid arc as per BP64 3.1 dn: cn=schema attributeTypes: ( 1.3.6.1.3.1.666.666.3.1 NAME 'accessKey' DESC 'Key for accessing data' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE ) attributeTypes: ( 1.3.6.1.3.1.666.666.3.2 NAME 'secretKey' DESC 'Secret key' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE ) -attributeTypes: ( 1.3.6.1.3.1.666.666.3.3 NAME 'keyFingerprint' DESC 'Fingerprint of private key' EQUALITY caseIgnoreMatch SUBSTR caseIgnoreSubstringsMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.15 SINGLE-VALUE) -attributeTypes: ( 1.3.6.1.3.1.666.666.3.4 NAME 'isAdmin' DESC 'Is user an administrator?' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) -attributeTypes: ( 1.3.6.1.3.1.666.666.3.5 NAME 'projectManager' DESC 'Project Managers of a project' SYNTAX 1.3.6.1.4.1.1466.115.121.1.12 ) -objectClasses: ( 1.3.6.1.3.1.666.666.4.1 NAME 'novaUser' DESC 'access and secret keys' SUP top AUXILIARY MUST ( uid ) MAY ( accessKey $ secretKey $ isAdmin ) ) -objectClasses: ( 1.3.6.1.3.1.666.666.4.2 NAME 'novaKeyPair' DESC 'Key pair for User' SUP top STRUCTURAL MUST ( cn $ sshPublicKey $ keyFingerprint ) ) -objectClasses: ( 1.3.6.1.3.1.666.666.4.3 NAME 'novaProject' DESC 'Container for project' SUP groupOfNames STRUCTURAL MUST ( cn $ projectManager ) ) +attributeTypes: ( 1.3.6.1.3.1.666.666.3.4 NAME 'isNovaAdmin' DESC 'Is user a nova administrator?' EQUALITY booleanMatch SYNTAX 1.3.6.1.4.1.1466.115.121.1.7 SINGLE-VALUE ) +objectClasses: ( 1.3.6.1.3.1.666.666.4.1 NAME 'novaUser' DESC 'access and secret keys' SUP top AUXILIARY MAY ( accessKey $ secretKey $ isNovaAdmin ) ) diff --git a/nova/auth/opendj.sh b/nova/auth/opendj.sh new file mode 100755 index 000000000..1a280e5a8 --- /dev/null +++ b/nova/auth/opendj.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# LDAP INSTALL SCRIPT - IS IDEMPOTENT, does not scrub users + +apt-get install -y ldap-utils python-ldap openjdk-6-jre + +if [ ! -d "/usr/opendj" ] +then + # TODO(rlane): Wikimedia Foundation is the current package maintainer. + # After the package is included in Ubuntu's channel, change this. + wget http://apt.wikimedia.org/wikimedia/pool/main/o/opendj/opendj_2.4.0-7_amd64.deb + dpkg -i opendj_2.4.0-7_amd64.deb +fi + +abspath=`dirname "$(cd "${0%/*}" 2>/dev/null; echo "$PWD"/"${0##*/}")"` +schemapath='/var/opendj/instance/config/schema' +cp $abspath/openssh-lpk_sun.schema $schemapath/97-openssh-lpk_sun.ldif +cp $abspath/nova_sun.schema $schemapath/98-nova_sun.ldif +chown opendj:opendj $schemapath/98-nova_sun.ldif + +cat >/etc/ldap/ldap.conf <<LDAP_CONF_EOF +# LDAP Client Settings +URI ldap://localhost +BASE dc=example,dc=com +BINDDN cn=Directory Manager +SIZELIMIT 0 +TIMELIMIT 0 +LDAP_CONF_EOF + +cat >/etc/ldap/base.ldif <<BASE_LDIF_EOF +# This is the root of the directory tree +dn: dc=example,dc=com +description: Example.Com, your trusted non-existent corporation. +dc: example +o: Example.Com +objectClass: top +objectClass: dcObject +objectClass: organization + +# Subtree for users +dn: ou=Users,dc=example,dc=com +ou: Users +description: Users +objectClass: organizationalUnit + +# Subtree for groups +dn: ou=Groups,dc=example,dc=com +ou: Groups +description: Groups +objectClass: organizationalUnit + +# Subtree for system accounts +dn: ou=System,dc=example,dc=com +ou: System +description: Special accounts used by software applications. +objectClass: organizationalUnit + +# Special Account for Authentication: +dn: uid=authenticate,ou=System,dc=example,dc=com +uid: authenticate +ou: System +description: Special account for authenticating users +userPassword: {MD5}TLnIqASP0CKUR3/LGkEZGg== +objectClass: account +objectClass: simpleSecurityObject + +# create the sysadmin entry + +dn: cn=developers,ou=Groups,dc=example,dc=com +objectclass: groupOfNames +cn: developers +description: IT admin group +member: uid=admin,ou=Users,dc=example,dc=com + +dn: cn=sysadmins,ou=Groups,dc=example,dc=com +objectclass: groupOfNames +cn: sysadmins +description: IT admin group +member: uid=admin,ou=Users,dc=example,dc=com + +dn: cn=netadmins,ou=Groups,dc=example,dc=com +objectclass: groupOfNames +cn: netadmins +description: Network admin group +member: uid=admin,ou=Users,dc=example,dc=com + +dn: cn=cloudadmins,ou=Groups,dc=example,dc=com +objectclass: groupOfNames +cn: cloudadmins +description: Cloud admin group +member: uid=admin,ou=Users,dc=example,dc=com + +dn: cn=itsec,ou=Groups,dc=example,dc=com +objectclass: groupOfNames +cn: itsec +description: IT security users group +member: uid=admin,ou=Users,dc=example,dc=com +BASE_LDIF_EOF + +/etc/init.d/opendj stop +su - opendj -c '/usr/opendj/setup -i -b "dc=example,dc=com" -l /etc/ldap/base.ldif -S -w changeme -O -n --noPropertiesFile' +/etc/init.d/opendj start diff --git a/nova/auth/slap.sh b/nova/auth/slap.sh index 797675d2e..95c61dafd 100755 --- a/nova/auth/slap.sh +++ b/nova/auth/slap.sh @@ -22,7 +22,7 @@ apt-get install -y slapd ldap-utils python-ldap abspath=`dirname "$(cd "${0%/*}" 2>/dev/null; echo "$PWD"/"${0##*/}")"` cp $abspath/openssh-lpk_openldap.schema /etc/ldap/schema/openssh-lpk_openldap.schema -cp $abspath/nova_openldap.schema /etc/ldap/schema/nova_openldap.schema +cp $abspath/nova_openldap.schema /etc/ldap/schema/nova.schema mv /etc/ldap/slapd.conf /etc/ldap/slapd.conf.orig cat >/etc/ldap/slapd.conf <<SLAPD_CONF_EOF @@ -33,7 +33,6 @@ cat >/etc/ldap/slapd.conf <<SLAPD_CONF_EOF include /etc/ldap/schema/core.schema include /etc/ldap/schema/cosine.schema include /etc/ldap/schema/inetorgperson.schema -include /etc/ldap/schema/openssh-lpk_openldap.schema include /etc/ldap/schema/nova.schema pidfile /var/run/slapd/slapd.pid argsfile /var/run/slapd/slapd.args diff --git a/nova/cloudpipe/bootscript.sh b/nova/cloudpipe/bootscript.sh deleted file mode 100755 index 30d9ad102..000000000 --- a/nova/cloudpipe/bootscript.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/bash -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This gets zipped and run on the cloudpipe-managed OpenVPN server - -export SUPERVISOR="http://10.255.255.1:8773/cloudpipe" -export VPN_IP=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 | awk '{print $1}'` -export BROADCAST=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f3 | awk '{print $1}'` -export DHCP_MASK=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f4 | awk '{print $1}'` -export GATEWAY=`netstat -r | grep default | cut -d' ' -f10` -export SUBJ="/C=US/ST=California/L=MountainView/O=AnsoLabs/OU=NovaDev/CN=customer-vpn-$VPN_IP" - -DHCP_LOWER=`echo $BROADCAST | awk -F. '{print $1"."$2"."$3"." $4 - 10 }'` -DHCP_UPPER=`echo $BROADCAST | awk -F. '{print $1"."$2"."$3"." $4 - 1 }'` - -# generate a server DH -openssl dhparam -out /etc/openvpn/dh1024.pem 1024 - -# generate a server priv key -openssl genrsa -out /etc/openvpn/server.key 2048 - -# generate a server CSR -openssl req -new -key /etc/openvpn/server.key -out /etc/openvpn/server.csr -batch -subj "$SUBJ" - -# URLEncode the CSR -CSRTEXT=`cat /etc/openvpn/server.csr` -CSRTEXT=$(python -c "import urllib; print urllib.quote('''$CSRTEXT''')") - -# SIGN the csr and save as server.crt -# CURL fetch to the supervisor, POSTing the CSR text, saving the result as the CRT file -curl --fail $SUPERVISOR -d "cert=$CSRTEXT" > /etc/openvpn/server.crt -curl --fail $SUPERVISOR/getca/ > /etc/openvpn/ca.crt - -# Customize the server.conf.template -cd /etc/openvpn - -sed -e s/VPN_IP/$VPN_IP/g server.conf.template > server.conf -sed -i -e s/DHCP_SUBNET/$DHCP_MASK/g server.conf -sed -i -e s/DHCP_LOWER/$DHCP_LOWER/g server.conf -sed -i -e s/DHCP_UPPER/$DHCP_UPPER/g server.conf -sed -i -e s/max-clients\ 1/max-clients\ 10/g server.conf - -echo "\npush \"route 10.255.255.1 255.255.255.255 $GATEWAY\"\n" >> server.conf -echo "\npush \"route 10.255.255.253 255.255.255.255 $GATEWAY\"\n" >> server.conf -echo "\nduplicate-cn\n" >> server.conf - -/etc/init.d/openvpn start diff --git a/nova/cloudpipe/bootscript.template b/nova/cloudpipe/bootscript.template new file mode 100755 index 000000000..94dea3f87 --- /dev/null +++ b/nova/cloudpipe/bootscript.template @@ -0,0 +1,51 @@ +#!/bin/bash +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This gets zipped and run on the cloudpipe-managed OpenVPN server + +export LC_ALL=C +export VPN_IP=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f2 | awk '{print $$1}'` +export BROADCAST=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f3 | awk '{print $$1}'` +export DHCP_MASK=`ifconfig | grep 'inet addr:'| grep -v '127.0.0.1' | cut -d: -f4 | awk '{print $$1}'` +export GATEWAY=`netstat -r | grep default | cut -d' ' -f10` + +DHCP_LOWER=`echo $$BROADCAST | awk -F. '{print $$1"."$$2"."$$3"." $$4 - ${num_vpn} }'` +DHCP_UPPER=`echo $$BROADCAST | awk -F. '{print $$1"."$$2"."$$3"." $$4 - 1 }'` + +# generate a server DH +openssl dhparam -out /etc/openvpn/dh1024.pem 1024 + +cp crl.pem /etc/openvpn/ +cp server.key /etc/openvpn/ +cp ca.crt /etc/openvpn/ +cp server.crt /etc/openvpn/ +# Customize the server.conf.template +cd /etc/openvpn + +sed -e s/VPN_IP/$$VPN_IP/g server.conf.template > server.conf +sed -i -e s/DHCP_SUBNET/$$DHCP_MASK/g server.conf +sed -i -e s/DHCP_LOWER/$$DHCP_LOWER/g server.conf +sed -i -e s/DHCP_UPPER/$$DHCP_UPPER/g server.conf +sed -i -e s/max-clients\ 1/max-clients\ 10/g server.conf + +echo "push \"route ${dmz_net} ${dmz_mask} $$GATEWAY\"" >> server.conf +echo "duplicate-cn" >> server.conf +echo "crl-verify /etc/openvpn/crl.pem" >> server.conf + +/etc/init.d/openvpn start diff --git a/nova/cloudpipe/pipelib.py b/nova/cloudpipe/pipelib.py index 3472201cd..09361828d 100644 --- a/nova/cloudpipe/pipelib.py +++ b/nova/cloudpipe/pipelib.py @@ -22,13 +22,15 @@ an instance with it. """ -import base64 import logging import os +import string import tempfile import zipfile from nova import context +from nova import crypto +from nova import db from nova import exception from nova import flags from nova import utils @@ -39,8 +41,17 @@ from nova.api.ec2 import cloud FLAGS = flags.FLAGS flags.DEFINE_string('boot_script_template', - utils.abspath('cloudpipe/bootscript.sh'), - 'Template for script to run on cloudpipe instance boot') + utils.abspath('cloudpipe/bootscript.template'), + _('Template for script to run on cloudpipe instance boot')) +flags.DEFINE_string('dmz_net', + '10.0.0.0', + _('Network to push into openvpn config')) +flags.DEFINE_string('dmz_mask', + '255.255.255.0', + _('Netmask to push into openvpn config')) + + +LOG = logging.getLogger('nova-cloudpipe') class CloudPipe(object): @@ -48,64 +59,96 @@ class CloudPipe(object): self.controller = cloud.CloudController() self.manager = manager.AuthManager() - def launch_vpn_instance(self, project_id): - logging.debug("Launching VPN for %s" % (project_id)) - project = self.manager.get_project(project_id) + def get_encoded_zip(self, project_id): # Make a payload.zip tmpfolder = tempfile.mkdtemp() filename = "payload.zip" zippath = os.path.join(tmpfolder, filename) z = zipfile.ZipFile(zippath, "w", zipfile.ZIP_DEFLATED) - - z.write(FLAGS.boot_script_template, 'autorun.sh') + shellfile = open(FLAGS.boot_script_template, "r") + s = string.Template(shellfile.read()) + shellfile.close() + boot_script = s.substitute(cc_dmz=FLAGS.cc_dmz, + cc_port=FLAGS.cc_port, + dmz_net=FLAGS.dmz_net, + dmz_mask=FLAGS.dmz_mask, + num_vpn=FLAGS.cnt_vpn_clients) + # genvpn, sign csr + crypto.generate_vpn_files(project_id) + z.writestr('autorun.sh', boot_script) + crl = os.path.join(crypto.ca_folder(project_id), 'crl.pem') + z.write(crl, 'crl.pem') + server_key = os.path.join(crypto.ca_folder(project_id), 'server.key') + z.write(server_key, 'server.key') + ca_crt = os.path.join(crypto.ca_path(project_id)) + z.write(ca_crt, 'ca.crt') + server_crt = os.path.join(crypto.ca_folder(project_id), 'server.crt') + z.write(server_crt, 'server.crt') z.close() - - key_name = self.setup_key_pair(project.project_manager_id, project_id) zippy = open(zippath, "r") - context = context.RequestContext(user=project.project_manager, - project=project) - - reservation = self.controller.run_instances(context, - # Run instances expects encoded userdata, it is decoded in the - # get_metadata_call. autorun.sh also decodes the zip file, hence - # the double encoding. - user_data=zippy.read().encode("base64").encode("base64"), + # NOTE(vish): run instances expects encoded userdata, it is decoded + # in the get_metadata_call. autorun.sh also decodes the zip file, + # hence the double encoding. + encoded = zippy.read().encode("base64").encode("base64") + zippy.close() + return encoded + + def launch_vpn_instance(self, project_id): + LOG.debug(_("Launching VPN for %s") % (project_id)) + project = self.manager.get_project(project_id) + ctxt = context.RequestContext(user=project.project_manager, + project=project) + key_name = self.setup_key_pair(ctxt) + group_name = self.setup_security_group(ctxt) + + reservation = self.controller.run_instances(ctxt, + user_data=self.get_encoded_zip(project_id), max_count=1, min_count=1, instance_type='m1.tiny', image_id=FLAGS.vpn_image_id, key_name=key_name, - security_groups=["vpn-secgroup"]) - zippy.close() + security_group=[group_name]) + + def setup_security_group(self, context): + group_name = '%s%s' % (context.project.id, FLAGS.vpn_key_suffix) + if db.security_group_exists(context, context.project.id, group_name): + return group_name + group = {'user_id': context.user.id, + 'project_id': context.project.id, + 'name': group_name, + 'description': 'Group for vpn'} + group_ref = db.security_group_create(context, group) + rule = {'parent_group_id': group_ref['id'], + 'cidr': '0.0.0.0/0', + 'protocol': 'udp', + 'from_port': 1194, + 'to_port': 1194} + db.security_group_rule_create(context, rule) + rule = {'parent_group_id': group_ref['id'], + 'cidr': '0.0.0.0/0', + 'protocol': 'icmp', + 'from_port': -1, + 'to_port': -1} + db.security_group_rule_create(context, rule) + # NOTE(vish): No need to trigger the group since the instance + # has not been run yet. + return group_name - def setup_key_pair(self, user_id, project_id): - key_name = '%s%s' % (project_id, FLAGS.vpn_key_suffix) + def setup_key_pair(self, context): + key_name = '%s%s' % (context.project.id, FLAGS.vpn_key_suffix) try: - private_key, fingerprint = self.manager.generate_key_pair(user_id, - key_name) + result = cloud._gen_key(context, context.user.id, key_name) + private_key = result['private_key'] try: - key_dir = os.path.join(FLAGS.keys_path, user_id) + key_dir = os.path.join(FLAGS.keys_path, context.user.id) if not os.path.exists(key_dir): os.makedirs(key_dir) - file_name = os.path.join(key_dir, '%s.pem' % key_name) - with open(file_name, 'w') as f: + key_path = os.path.join(key_dir, '%s.pem' % key_name) + with open(key_path, 'w') as f: f.write(private_key) except: pass except exception.Duplicate: pass return key_name - - # def setup_secgroups(self, username): - # conn = self.euca.connection_for(username) - # try: - # secgroup = conn.create_security_group("vpn-secgroup", - # "vpn-secgroup") - # secgroup.authorize(ip_protocol = "udp", from_port = "1194", - # to_port = "1194", cidr_ip = "0.0.0.0/0") - # secgroup.authorize(ip_protocol = "tcp", from_port = "80", - # to_port = "80", cidr_ip = "0.0.0.0/0") - # secgroup.authorize(ip_protocol = "tcp", from_port = "22", - # to_port = "22", cidr_ip = "0.0.0.0/0") - # except: - # pass diff --git a/nova/compute/api.py b/nova/compute/api.py index 929342a1e..038f52046 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -20,6 +20,7 @@ Handles all API requests relating to instances (guest vms). """ +import datetime import logging import time @@ -43,17 +44,37 @@ def generate_default_hostname(internal_id): class ComputeAPI(base.Base): """API for interacting with the compute manager.""" - def __init__(self, **kwargs): - self.network_manager = utils.import_object(FLAGS.network_manager) + def __init__(self, network_manager=None, image_service=None, **kwargs): + if not network_manager: + network_manager = utils.import_object(FLAGS.network_manager) + self.network_manager = network_manager + if not image_service: + image_service = utils.import_object(FLAGS.image_service) + self.image_service = image_service super(ComputeAPI, self).__init__(**kwargs) - # TODO(eday): network_topic arg should go away once we push network - # allocation into the scheduler or compute worker. - def create_instances(self, context, instance_type, image_service, image_id, - network_topic, min_count=1, max_count=1, - kernel_id=None, ramdisk_id=None, name='', - description='', user_data='', key_name=None, + def get_network_topic(self, context, instance_id): + try: + instance = self.db.instance_get_by_internal_id(context, + instance_id) + except exception.NotFound as e: + logging.warning("Instance %d was not found in get_network_topic", + instance_id) + raise e + + host = instance['host'] + if not host: + raise exception.Error("Instance %d has no host" % instance_id) + topic = self.db.queue_get_for(context, FLAGS.compute_topic, host) + return rpc.call(context, + topic, + {"method": "get_network_topic", "args": {'fake': 1}}) + + def create_instances(self, context, instance_type, image_id, min_count=1, + max_count=1, kernel_id=None, ramdisk_id=None, + display_name='', description='', key_name=None, key_data=None, security_group='default', + user_data=None, generate_hostname=generate_default_hostname): """Create the number of instances requested if quote and other arguments check out ok.""" @@ -69,15 +90,20 @@ class ComputeAPI(base.Base): is_vpn = image_id == FLAGS.vpn_image_id if not is_vpn: - image = image_service.show(context, image_id) + image = self.image_service.show(context, image_id) + + # If kernel_id/ramdisk_id isn't explicitly set in API call + # we take the defaults from the image's metadata if kernel_id is None: - kernel_id = image.get('kernelId', FLAGS.default_kernel) + kernel_id = image.get('kernelId', None) if ramdisk_id is None: - ramdisk_id = image.get('ramdiskId', FLAGS.default_ramdisk) + ramdisk_id = image.get('ramdiskId', None) # Make sure we have access to kernel and ramdisk - image_service.show(context, kernel_id) - image_service.show(context, ramdisk_id) + if kernel_id: + self.image_service.show(context, kernel_id) + if ramdisk_id: + self.image_service.show(context, ramdisk_id) if security_group is None: security_group = ['default'] @@ -100,8 +126,8 @@ class ComputeAPI(base.Base): base_options = { 'reservation_id': utils.generate_uid('r'), 'image_id': image_id, - 'kernel_id': kernel_id, - 'ramdisk_id': ramdisk_id, + 'kernel_id': kernel_id or '', + 'ramdisk_id': ramdisk_id or '', 'state_description': 'scheduling', 'user_id': context.user_id, 'project_id': context.project_id, @@ -110,41 +136,41 @@ class ComputeAPI(base.Base): 'memory_mb': type_data['memory_mb'], 'vcpus': type_data['vcpus'], 'local_gb': type_data['local_gb'], - 'display_name': name, + 'display_name': display_name, 'display_description': description, + 'user_data': user_data or '', 'key_name': key_name, 'key_data': key_data} elevated = context.elevated() instances = [] - logging.debug("Going to run %s instances...", num_instances) + logging.debug(_("Going to run %s instances..."), num_instances) for num in range(num_instances): instance = dict(mac_address=utils.generate_mac(), launch_index=num, **base_options) - instance_ref = self.create_instance(context, security_groups, - **instance) - instance_id = instance_ref['id'] - internal_id = instance_ref['internal_id'] - hostname = generate_hostname(internal_id) - self.update_instance(context, instance_id, hostname=hostname) - instances.append(dict(id=instance_id, internal_id=internal_id, - hostname=hostname, **instance)) - - # TODO(vish): This probably should be done in the scheduler - # or in compute as a call. The network should be - # allocated after the host is assigned and setup - # can happen at the same time. - address = self.network_manager.allocate_fixed_ip(context, - instance_id, - is_vpn) - rpc.cast(elevated, - network_topic, - {"method": "setup_fixed_ip", - "args": {"address": address}}) - - logging.debug("Casting to scheduler for %s/%s's instance %s" % - (context.project_id, context.user_id, instance_id)) + instance = self.db.instance_create(context, instance) + instance_id = instance['id'] + internal_id = instance['internal_id'] + + elevated = context.elevated() + if not security_groups: + security_groups = [] + for security_group_id in security_groups: + self.db.instance_add_security_group(elevated, + instance_id, + security_group_id) + + # Set sane defaults if not specified + updates = dict(hostname=generate_hostname(internal_id)) + if 'display_name' not in instance: + updates['display_name'] = "Server %s" % internal_id + + instance = self.update_instance(context, instance_id, **updates) + instances.append(instance) + + logging.debug(_("Casting to scheduler for %s/%s's instance %s"), + context.project_id, context.user_id, instance_id) rpc.cast(context, FLAGS.scheduler_topic, {"method": "run_instance", @@ -154,6 +180,12 @@ class ComputeAPI(base.Base): return instances def ensure_default_security_group(self, context): + """ Create security group for the security context if it + does not already exist + + :param context: the security context + + """ try: db.security_group_get_by_name(context, context.project_id, 'default') @@ -162,40 +194,7 @@ class ComputeAPI(base.Base): 'description': 'default', 'user_id': context.user_id, 'project_id': context.project_id} - group = db.security_group_create(context, values) - - def create_instance(self, context, security_groups=None, **kwargs): - """Creates the instance in the datastore and returns the - new instance as a mapping - - :param context: The security context - :param security_groups: list of security group ids to - attach to the instance - :param kwargs: All additional keyword args are treated - as data fields of the instance to be - created - - :retval Returns a mapping of the instance information - that has just been created - - """ - instance_ref = self.db.instance_create(context, kwargs) - inst_id = instance_ref['id'] - # Set sane defaults if not specified - if kwargs.get('display_name') is None: - display_name = "Server %s" % instance_ref['internal_id'] - instance_ref['display_name'] = display_name - self.db.instance_update(context, inst_id, - {'display_name': display_name}) - - elevated = context.elevated() - if not security_groups: - security_groups = [] - for security_group_id in security_groups: - self.db.instance_add_security_group(elevated, - inst_id, - security_group_id) - return instance_ref + db.security_group_create(context, values) def update_instance(self, context, instance_id, **kwargs): """Updates the instance in the datastore. @@ -209,4 +208,96 @@ class ComputeAPI(base.Base): :retval None """ - self.db.instance_update(context, instance_id, kwargs) + return self.db.instance_update(context, instance_id, kwargs) + + def delete_instance(self, context, instance_id): + logging.debug("Going to try and terminate %d" % instance_id) + try: + instance = self.db.instance_get_by_internal_id(context, + instance_id) + except exception.NotFound as e: + logging.warning(_("Instance %d was not found during terminate"), + instance_id) + raise e + + if (instance['state_description'] == 'terminating'): + logging.warning(_("Instance %d is already being terminated"), + instance_id) + return + + self.update_instance(context, + instance['id'], + state_description='terminating', + state=0, + terminated_at=datetime.datetime.utcnow()) + + host = instance['host'] + logging.error('terminate %s %s %s %s',context, FLAGS.compute_topic, host, self.db.queue_get_for(context, FLAGS.compute_topic, host)) + if host: + rpc.cast(context, + self.db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "terminate_instance", + "args": {"instance_id": instance['id']}}) + else: + self.db.instance_destroy(context, instance['id']) + + def get_instances(self, context, project_id=None): + """Get all instances, possibly filtered by project ID or + user ID. If there is no filter and the context is an admin, + it will retreive all instances in the system.""" + if project_id or not context.is_admin: + if not context.project: + return self.db.instance_get_all_by_user(context, + context.user_id) + if project_id is None: + project_id = context.project_id + return self.db.instance_get_all_by_project(context, project_id) + return self.db.instance_get_all(context) + + def get_instance(self, context, instance_id): + return self.db.instance_get_by_internal_id(context, instance_id) + + def reboot(self, context, instance_id): + """Reboot the given instance.""" + instance = self.db.instance_get_by_internal_id(context, instance_id) + host = instance['host'] + rpc.cast(context, + self.db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "reboot_instance", + "args": {"instance_id": instance['id']}}) + + def pause(self, context, instance_id): + """Pause the given instance.""" + instance = self.db.instance_get_by_internal_id(context, instance_id) + host = instance['host'] + rpc.cast(context, + self.db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "pause_instance", + "args": {"instance_id": instance['id']}}) + + def unpause(self, context, instance_id): + """Unpause the given instance.""" + instance = self.db.instance_get_by_internal_id(context, instance_id) + host = instance['host'] + rpc.cast(context, + self.db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "unpause_instance", + "args": {"instance_id": instance['id']}}) + + def rescue(self, context, instance_id): + """Rescue the given instance.""" + instance = self.db.instance_get_by_internal_id(context, instance_id) + host = instance['host'] + rpc.cast(context, + self.db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "rescue_instance", + "args": {"instance_id": instance['id']}}) + + def unrescue(self, context, instance_id): + """Unrescue the given instance.""" + instance = self.db.instance_get_by_internal_id(context, instance_id) + host = instance['host'] + rpc.cast(context, + self.db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "unrescue_instance", + "args": {"instance_id": instance['id']}}) diff --git a/nova/compute/disk.py b/nova/compute/disk.py index 4338d39f0..814a258cd 100644 --- a/nova/compute/disk.py +++ b/nova/compute/disk.py @@ -26,8 +26,6 @@ import logging import os import tempfile -from twisted.internet import defer - from nova import exception from nova import flags @@ -39,7 +37,6 @@ flags.DEFINE_integer('block_size', 1024 * 1024 * 256, 'block_size to use for dd') -@defer.inlineCallbacks def partition(infile, outfile, local_bytes=0, resize=True, local_type='ext2', execute=None): """ @@ -64,18 +61,18 @@ def partition(infile, outfile, local_bytes=0, resize=True, file_size = os.path.getsize(infile) if resize and file_size < FLAGS.minimum_root_size: last_sector = FLAGS.minimum_root_size / sector_size - 1 - yield execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' - % (infile, last_sector, sector_size)) - yield execute('e2fsck -fp %s' % infile, check_exit_code=False) - yield execute('resize2fs %s' % infile) + execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' + % (infile, last_sector, sector_size)) + execute('e2fsck -fp %s' % infile, check_exit_code=False) + execute('resize2fs %s' % infile) file_size = FLAGS.minimum_root_size elif file_size % sector_size != 0: - logging.warn("Input partition size not evenly divisible by" - " sector size: %d / %d", file_size, sector_size) + logging.warn(_("Input partition size not evenly divisible by" + " sector size: %d / %d"), file_size, sector_size) primary_sectors = file_size / sector_size if local_bytes % sector_size != 0: - logging.warn("Bytes for local storage not evenly divisible" - " by sector size: %d / %d", local_bytes, sector_size) + logging.warn(_("Bytes for local storage not evenly divisible" + " by sector size: %d / %d"), local_bytes, sector_size) local_sectors = local_bytes / sector_size mbr_last = 62 # a @@ -86,30 +83,36 @@ def partition(infile, outfile, local_bytes=0, resize=True, last_sector = local_last # e # create an empty file - yield execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' - % (outfile, mbr_last, sector_size)) + execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' + % (outfile, mbr_last, sector_size)) # make mbr partition - yield execute('parted --script %s mklabel msdos' % outfile) + execute('parted --script %s mklabel msdos' % outfile) # append primary file - yield execute('dd if=%s of=%s bs=%s conv=notrunc,fsync oflag=append' - % (infile, outfile, FLAGS.block_size)) + execute('dd if=%s of=%s bs=%s conv=notrunc,fsync oflag=append' + % (infile, outfile, FLAGS.block_size)) # make primary partition - yield execute('parted --script %s mkpart primary %ds %ds' - % (outfile, primary_first, primary_last)) + execute('parted --script %s mkpart primary %ds %ds' + % (outfile, primary_first, primary_last)) if local_bytes > 0: # make the file bigger - yield execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' - % (outfile, last_sector, sector_size)) + execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' + % (outfile, last_sector, sector_size)) # make and format local partition - yield execute('parted --script %s mkpartfs primary %s %ds %ds' - % (outfile, local_type, local_first, local_last)) + execute('parted --script %s mkpartfs primary %s %ds %ds' + % (outfile, local_type, local_first, local_last)) + + +def extend(image, size, execute): + file_size = os.path.getsize(image) + if file_size >= size: + return + return execute('truncate -s size %s' % (image,)) -@defer.inlineCallbacks def inject_data(image, key=None, net=None, partition=None, execute=None): """Injects a ssh key and optionally net data into a disk image. @@ -119,74 +122,83 @@ def inject_data(image, key=None, net=None, partition=None, execute=None): If partition is not specified it mounts the image as a single partition. """ - out, err = yield execute('sudo losetup -f --show %s' % image) + out, err = execute('sudo losetup --find --show %s' % image) if err: - raise exception.Error('Could not attach image to loopback: %s' % err) + raise exception.Error(_('Could not attach image to loopback: %s') + % err) device = out.strip() try: if not partition is None: # create partition - out, err = yield execute('sudo kpartx -a %s' % device) + out, err = execute('sudo kpartx -a %s' % device) if err: - raise exception.Error('Failed to load partition: %s' % err) + raise exception.Error(_('Failed to load partition: %s') % err) mapped_device = '/dev/mapper/%sp%s' % (device.split('/')[-1], partition) else: mapped_device = device - out, err = yield execute('sudo tune2fs -c 0 -i 0 %s' % mapped_device) + + # We can only loopback mount raw images. If the device isn't there, + # it's normally because it's a .vmdk or a .vdi etc + if not os.path.exists(mapped_device): + raise exception.Error('Mapped device was not found (we can' + ' only inject raw disk images): %s' % + mapped_device) + + # Configure ext2fs so that it doesn't auto-check every N boots + out, err = execute('sudo tune2fs -c 0 -i 0 %s' % mapped_device) tmpdir = tempfile.mkdtemp() try: # mount loopback to dir - out, err = yield execute( + out, err = execute( 'sudo mount %s %s' % (mapped_device, tmpdir)) if err: - raise exception.Error('Failed to mount filesystem: %s' % err) + raise exception.Error(_('Failed to mount filesystem: %s') + % err) try: if key: # inject key file - yield _inject_key_into_fs(key, tmpdir, execute=execute) + _inject_key_into_fs(key, tmpdir, execute=execute) if net: - yield _inject_net_into_fs(net, tmpdir, execute=execute) + _inject_net_into_fs(net, tmpdir, execute=execute) finally: # unmount device - yield execute('sudo umount %s' % mapped_device) + execute('sudo umount %s' % mapped_device) finally: # remove temporary directory - yield execute('rmdir %s' % tmpdir) + execute('rmdir %s' % tmpdir) if not partition is None: # remove partitions - yield execute('sudo kpartx -d %s' % device) + execute('sudo kpartx -d %s' % device) finally: # remove loopback - yield execute('sudo losetup -d %s' % device) + execute('sudo losetup --detach %s' % device) -@defer.inlineCallbacks def _inject_key_into_fs(key, fs, execute=None): """Add the given public ssh key to root's authorized_keys. key is an ssh key string. fs is the path to the base of the filesystem into which to inject the key. """ - sshdir = os.path.join(os.path.join(fs, 'root'), '.ssh') - yield execute('sudo mkdir -p %s' % sshdir) # existing dir doesn't matter - yield execute('sudo chown root %s' % sshdir) - yield execute('sudo chmod 700 %s' % sshdir) + sshdir = os.path.join(fs, 'root', '.ssh') + execute('sudo mkdir -p %s' % sshdir) # existing dir doesn't matter + execute('sudo chown root %s' % sshdir) + execute('sudo chmod 700 %s' % sshdir) keyfile = os.path.join(sshdir, 'authorized_keys') - yield execute('sudo tee -a %s' % keyfile, '\n' + key.strip() + '\n') + execute('sudo tee -a %s' % keyfile, '\n' + key.strip() + '\n') -@defer.inlineCallbacks def _inject_net_into_fs(net, fs, execute=None): """Inject /etc/network/interfaces into the filesystem rooted at fs. net is the contents of /etc/network/interfaces. """ netdir = os.path.join(os.path.join(fs, 'etc'), 'network') - yield execute('sudo mkdir -p %s' % netdir) # existing dir doesn't matter - yield execute('sudo chown root:root %s' % netdir) - yield execute('sudo chmod 755 %s' % netdir) + execute('sudo mkdir -p %s' % netdir) # existing dir doesn't matter + execute('sudo chown root:root %s' % netdir) + execute('sudo chmod 755 %s' % netdir) netfile = os.path.join(netdir, 'interfaces') - yield execute('sudo tee %s' % netfile, net) + execute('sudo tee %s' % netfile, net) diff --git a/nova/compute/instance_types.py b/nova/compute/instance_types.py index a2679e0fc..196d6a8df 100644 --- a/nova/compute/instance_types.py +++ b/nova/compute/instance_types.py @@ -22,6 +22,7 @@ The built-in instance properties. """ from nova import flags +from nova import exception FLAGS = flags.FLAGS INSTANCE_TYPES = { @@ -37,7 +38,7 @@ def get_by_type(instance_type): if instance_type is None: return FLAGS.default_instance_type if instance_type not in INSTANCE_TYPES: - raise exception.ApiError("Unknown instance type: %s", + raise exception.ApiError(_("Unknown instance type: %s"), instance_type) return instance_type diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 66bbb8d5a..dc660d2a6 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -42,11 +42,10 @@ import os import time -from twisted.internet import defer - from nova import exception from nova import flags from nova import manager +from nova import rpc from nova import utils from nova.compute import power_state from nova import rpc @@ -57,8 +56,10 @@ flags.DEFINE_string('instances_path', '$state_path/instances', 'where instances are stored on disk') flags.DEFINE_string('compute_driver', 'nova.virt.connection.get_connection', 'Driver to use for controlling virtualization') -flags.DEFINE_string('live_migration_timeout', 30, +flags.DEFINE_string('live_migration_timeout', 10, 'Timeout value for pre_live_migration is completed.') +flags.DEFINE_string('stub_network', False, + 'Stub network related code') class ComputeManager(manager.Manager): @@ -76,6 +77,12 @@ class ComputeManager(manager.Manager): self.volume_manager = utils.import_object(FLAGS.volume_manager) super(ComputeManager, self).__init__(*args, **kwargs) + def init_host(self): + """Do any initialization that needs to be run if this is a + standalone service. + """ + self.driver.init_host() + def _update_state(self, context, instance_id): """Update the state of an instance from the driver info.""" # FIXME(ja): include other fields from state? @@ -87,26 +94,56 @@ class ComputeManager(manager.Manager): state = power_state.NOSTATE self.db.instance_set_state(context, instance_id, state) - @defer.inlineCallbacks + def get_network_topic(self, context, **_kwargs): + """Retrieves the network host for a project on this host""" + # TODO(vish): This method should be memoized. This will make + # the call to get_network_host cheaper, so that + # it can pas messages instead of checking the db + # locally. + if FLAGS.stub_network: + host = FLAGS.network_host + else: + host = self.network_manager.get_network_host(context) + return self.db.queue_get_for(context, + FLAGS.network_topic, + host) + @exception.wrap_exception def refresh_security_group(self, context, security_group_id, **_kwargs): """This call passes stright through to the virtualization driver.""" - yield self.driver.refresh_security_group(security_group_id) + self.driver.refresh_security_group(security_group_id) - @defer.inlineCallbacks @exception.wrap_exception def run_instance(self, context, instance_id, **_kwargs): """Launch a new instance with specified options.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) if instance_ref['name'] in self.driver.list_instances(): - raise exception.Error("Instance has already been created") - logging.debug("instance %s: starting...", instance_id) - self.network_manager.setup_compute_network(context, instance_id) + raise exception.Error(_("Instance has already been created")) self.db.instance_update(context, instance_id, {'host': self.host, 'launched_on':self.host}) + self.db.instance_set_state(context, + instance_id, + power_state.NOSTATE, + 'networking') + + is_vpn = instance_ref['image_id'] == FLAGS.vpn_image_id + # NOTE(vish): This could be a cast because we don't do anything + # with the address currently, but I'm leaving it as + # a call to ensure that network setup completes. We + # will eventually also need to save the address here. + if not FLAGS.stub_network: + address = rpc.call(context, + self.get_network_topic(context), + {"method": "allocate_fixed_ip", + "args": {"instance_id": instance_id, + "vpn": is_vpn}}) + + self.network_manager.setup_compute_network(context, + instance_id) + # TODO(vish) check to make sure the availability zone matches self.db.instance_set_state(context, instance_id, @@ -114,13 +151,13 @@ class ComputeManager(manager.Manager): 'spawning') try: - yield self.driver.spawn(instance_ref) + self.driver.spawn(instance_ref) now = datetime.datetime.utcnow() self.db.instance_update(context, instance_id, {'launched_at': now}) except Exception: # pylint: disable-msg=W0702 - logging.exception("instance %s: Failed to spawn", + logging.exception(_("instance %s: Failed to spawn"), instance_ref['name']) self.db.instance_set_state(context, instance_id, @@ -128,104 +165,167 @@ class ComputeManager(manager.Manager): self._update_state(context, instance_id) - @defer.inlineCallbacks @exception.wrap_exception def terminate_instance(self, context, instance_id): """Terminate an instance on this machine.""" context = context.elevated() - logging.debug("instance %s: terminating", instance_id) instance_ref = self.db.instance_get(context, instance_id) + + if not FLAGS.stub_network: + address = self.db.instance_get_floating_address(context, + instance_ref['id']) + if address: + logging.debug(_("Disassociating address %s") % address) + # NOTE(vish): Right now we don't really care if the ip is + # disassociated. We may need to worry about + # checking this later. + rpc.cast(context, + self.get_network_topic(context), + {"method": "disassociate_floating_ip", + "args": {"floating_address": address}}) + + address = self.db.instance_get_fixed_address(context, + instance_ref['id']) + if address: + logging.debug(_("Deallocating address %s") % address) + # NOTE(vish): Currently, nothing needs to be done on the + # network node until release. If this changes, + # we will need to cast here. + self.network_manager.deallocate_fixed_ip(context.elevated(), + address) + + logging.debug(_("instance %s: terminating"), instance_id) + volumes = instance_ref.get('volumes', []) or [] for volume in volumes: self.detach_volume(context, instance_id, volume['id']) if instance_ref['state'] == power_state.SHUTOFF: self.db.instance_destroy(context, instance_id) - raise exception.Error('trying to destroy already destroyed' - ' instance: %s' % instance_id) - yield self.driver.destroy(instance_ref) + raise exception.Error(_('trying to destroy already destroyed' + ' instance: %s') % instance_id) + self.driver.destroy(instance_ref) # TODO(ja): should we keep it in a terminated state for a bit? self.db.instance_destroy(context, instance_id) - @defer.inlineCallbacks @exception.wrap_exception def reboot_instance(self, context, instance_id): """Reboot an instance on this server.""" context = context.elevated() - instance_ref = self.db.instance_get(context, instance_id) self._update_state(context, instance_id) + instance_ref = self.db.instance_get(context, instance_id) if instance_ref['state'] != power_state.RUNNING: - logging.warn('trying to reboot a non-running ' - 'instance: %s (state: %s excepted: %s)', + logging.warn(_('trying to reboot a non-running ' + 'instance: %s (state: %s excepted: %s)'), instance_ref['internal_id'], instance_ref['state'], power_state.RUNNING) - logging.debug('instance %s: rebooting', instance_ref['name']) + logging.debug(_('instance %s: rebooting'), instance_ref['name']) self.db.instance_set_state(context, instance_id, power_state.NOSTATE, 'rebooting') - yield self.driver.reboot(instance_ref) + self.network_manager.setup_compute_network(context, instance_id) + self.driver.reboot(instance_ref) self._update_state(context, instance_id) - @defer.inlineCallbacks @exception.wrap_exception def rescue_instance(self, context, instance_id): """Rescue an instance on this server.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) - logging.debug('instance %s: rescuing', + logging.debug(_('instance %s: rescuing'), instance_ref['internal_id']) self.db.instance_set_state(context, instance_id, power_state.NOSTATE, 'rescuing') - yield self.driver.rescue(instance_ref) + self.network_manager.setup_compute_network(context, instance_id) + self.driver.rescue(instance_ref) self._update_state(context, instance_id) - @defer.inlineCallbacks @exception.wrap_exception def unrescue_instance(self, context, instance_id): """Rescue an instance on this server.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) - logging.debug('instance %s: unrescuing', + logging.debug(_('instance %s: unrescuing'), instance_ref['internal_id']) self.db.instance_set_state(context, instance_id, power_state.NOSTATE, 'unrescuing') - yield self.driver.unrescue(instance_ref) + self.driver.unrescue(instance_ref) self._update_state(context, instance_id) + @staticmethod + def _update_state_callback(self, context, instance_id, result): + """Update instance state when async task completes.""" + self._update_state(context, instance_id) + + @exception.wrap_exception + def pause_instance(self, context, instance_id): + """Pause an instance on this server.""" + context = context.elevated() + instance_ref = self.db.instance_get(context, instance_id) + + logging.debug('instance %s: pausing', + instance_ref['internal_id']) + self.db.instance_set_state(context, + instance_id, + power_state.NOSTATE, + 'pausing') + self.driver.pause(instance_ref, + lambda result: self._update_state_callback(self, + context, + instance_id, + result)) + + @exception.wrap_exception + def unpause_instance(self, context, instance_id): + """Unpause a paused instance on this server.""" + context = context.elevated() + instance_ref = self.db.instance_get(context, instance_id) + + logging.debug('instance %s: unpausing', + instance_ref['internal_id']) + self.db.instance_set_state(context, + instance_id, + power_state.NOSTATE, + 'unpausing') + self.driver.unpause(instance_ref, + lambda result: self._update_state_callback(self, + context, + instance_id, + result)) + @exception.wrap_exception def get_console_output(self, context, instance_id): """Send the console output for an instance.""" context = context.elevated() - logging.debug("instance %s: getting console output", instance_id) + logging.debug(_("instance %s: getting console output"), instance_id) instance_ref = self.db.instance_get(context, instance_id) return self.driver.get_console_output(instance_ref) - @defer.inlineCallbacks @exception.wrap_exception def attach_volume(self, context, instance_id, volume_id, mountpoint): """Attach a volume to an instance.""" context = context.elevated() - logging.debug("instance %s: attaching volume %s to %s", instance_id, + logging.debug(_("instance %s: attaching volume %s to %s"), instance_id, volume_id, mountpoint) instance_ref = self.db.instance_get(context, instance_id) - dev_path = yield self.volume_manager.setup_compute_volume(context, - volume_id) + dev_path = self.volume_manager.setup_compute_volume(context, + volume_id) try: - yield self.driver.attach_volume(instance_ref['name'], - dev_path, - mountpoint) + self.driver.attach_volume(instance_ref['name'], + dev_path, + mountpoint) self.db.volume_attached(context, volume_id, instance_id, @@ -234,36 +334,35 @@ class ComputeManager(manager.Manager): # NOTE(vish): The inline callback eats the exception info so we # log the traceback here and reraise the same # ecxception below. - logging.exception("instance %s: attach failed %s, removing", + logging.exception(_("instance %s: attach failed %s, removing"), instance_id, mountpoint) - yield self.volume_manager.remove_compute_volume(context, - volume_id) + self.volume_manager.remove_compute_volume(context, + volume_id) raise exc - defer.returnValue(True) - @defer.inlineCallbacks + return True + @exception.wrap_exception def detach_volume(self, context, instance_id, volume_id): """Detach a volume from an instance.""" context = context.elevated() - logging.debug("instance %s: detaching volume %s", + logging.debug(_("instance %s: detaching volume %s"), instance_id, volume_id) instance_ref = self.db.instance_get(context, instance_id) volume_ref = self.db.volume_get(context, volume_id) if instance_ref['name'] not in self.driver.list_instances(): - logging.warn("Detaching volume from unknown instance %s", + logging.warn(_("Detaching volume from unknown instance %s"), instance_ref['name']) else: - yield self.driver.detach_volume(instance_ref['name'], - volume_ref['mountpoint']) - yield self.volume_manager.remove_compute_volume(context, volume_id) + self.driver.detach_volume(instance_ref['name'], + volume_ref['mountpoint']) + self.volume_manager.remove_compute_volume(context, volume_id) self.db.volume_detached(context, volume_id) - defer.returnValue(True) + return True def compareCPU(self, context, xml): """ Check the host cpu is compatible to a cpu given by xml.""" - logging.warn('good!') return self.driver.compareCPU(xml) def get_memory_mb(self): @@ -281,7 +380,7 @@ class ComputeManager(manager.Manager): def pre_live_migration(self, context, instance_id, dest): """Any preparation for live migration at dst host.""" - # 1. getting volume info ( shlf/slot number ) + # Getting volume info ( shlf/slot number ) instance_ref = db.instance_get(context, instance_id) ec2_id = instance_ref['hostname'] @@ -289,40 +388,35 @@ class ComputeManager(manager.Manager): try: volumes = db.volume_get_by_ec2_id(context, ec2_id) except exception.NotFound: - logging.debug('%s has no volume.', ec2_id) + logging.info(_('%s has no volume.'), ec2_id) shelf_slots = {} for vol in volumes: shelf, slot = db.volume_get_shelf_and_blade(context, vol['id']) shelf_slots[vol.id] = (shelf, slot) - # 2. getting fixed ips + # Getting fixed ips fixed_ip = db.instance_get_fixed_address(context, instance_id) if None == fixed_ip: exc_type = 'NotFoundError' - val = '%s(%s) doesnt have fixed_ip ' % (instance_id, ec2_id) + val = _('%s(%s) doesnt have fixed_ip') % (instance_id, ec2_id) tb = ''.join(traceback.format_tb(sys.exc_info()[2])) raise rpc.RemoteError(exc_type, val, tb) - # 3. if any volume is mounted, prepare here. + # If any volume is mounted, prepare here. if 0 != len(shelf_slots): pass - # 4. Creating nova-instance-instance-xxx, this is written to libvirt.xml, - # and can be seen when executin "virsh nwfiter-list" On destination host, - # this nwfilter is necessary. - # In addition this method is creating security rule ingress rule onto - # destination host. + # Creating nova-instance-instance-xxx, this is written to libvirt.xml, + # and can be seen when executin "virsh nwfiter-list" On destination host, + # this nwfilter is necessary. + # In addition this method is creating security rule ingress rule onto + # destination host. self.driver.setup_nwfilters_for_instance(instance_ref) # 5. bridge settings -<<<<<<< TREE - self.network_manager.setup_compute_network(instance_id) - return True -======= self.network_manager.setup_compute_network(context, instance_id) return True ->>>>>>> MERGE-SOURCE def nwfilter_for_instance_exists(self, context, instance_id): """Check nova-instance-instance-xxx filter exists """ @@ -332,7 +426,7 @@ class ComputeManager(manager.Manager): def live_migration(self, context, instance_id, dest): """executes live migration.""" - # 1. ask dest host to preparing live migration. + # Asking dest host to preparing live migration. compute_topic = db.queue_get_for(context, FLAGS.compute_topic, dest) ret = rpc.call(context, compute_topic, @@ -341,16 +435,16 @@ class ComputeManager(manager.Manager): 'dest': dest}}) if True != ret: - logging.error('Live migration failed(err at %s)', dest) + logging.error(_('Pre live migration failed(err at %s)'), dest) db.instance_set_state(context, instance_id, power_state.RUNNING, 'running') return - # waiting for setting up nwfilter(nova-instance-instance-xxx) + # Waiting for setting up nwfilter such as, nova-instance-instance-xxx. # otherwise, live migration fail. - timeout_count = range(FLAGS.live_migration_timeout) + timeout_count = range(FLAGS.live_migration_timeout * 2) while 0 != len(timeout_count): ret = rpc.call(context, compute_topic, @@ -360,17 +454,14 @@ class ComputeManager(manager.Manager): break timeout_count.pop() - time.sleep(1) + time.sleep(0.5) if not ret: - logging.error('Timeout for pre_live_migration at %s', dest) + logging.error(_('Timeout for pre_live_migration at %s'), dest) return - # 2. executing live migration + # Executing live migration # live_migration might raises ProcessExecution error, but # nothing must be recovered in this version. instance_ref = db.instance_get(context, instance_id) - ret = self.driver.live_migration(instance_ref, dest) - if not ret: - logging.debug('Fail to live migration') - return + self.driver.live_migration(context, instance_ref, dest) diff --git a/nova/compute/monitor.py b/nova/compute/monitor.py index 22653113a..60c347a5e 100644 --- a/nova/compute/monitor.py +++ b/nova/compute/monitor.py @@ -255,7 +255,7 @@ class Instance(object): Updates the instances statistics and stores the resulting graphs in the internal object store on the cloud controller. """ - logging.debug('updating %s...', self.instance_id) + logging.debug(_('updating %s...'), self.instance_id) try: data = self.fetch_cpu_stats() @@ -285,7 +285,7 @@ class Instance(object): graph_disk(self, '1w') graph_disk(self, '1m') except Exception: - logging.exception('unexpected error during update') + logging.exception(_('unexpected error during update')) self.last_updated = utcnow() @@ -351,7 +351,7 @@ class Instance(object): rd += rd_bytes wr += wr_bytes except TypeError: - logging.error('Cannot get blockstats for "%s" on "%s"', + logging.error(_('Cannot get blockstats for "%s" on "%s"'), disk, self.instance_id) raise @@ -373,7 +373,7 @@ class Instance(object): rx += stats[0] tx += stats[4] except TypeError: - logging.error('Cannot get ifstats for "%s" on "%s"', + logging.error(_('Cannot get ifstats for "%s" on "%s"'), interface, self.instance_id) raise @@ -408,7 +408,7 @@ class InstanceMonitor(object, service.Service): try: conn = virt_connection.get_connection(read_only=True) except Exception, exn: - logging.exception('unexpected exception getting connection') + logging.exception(_('unexpected exception getting connection')) time.sleep(FLAGS.monitoring_instances_delay) return @@ -423,7 +423,7 @@ class InstanceMonitor(object, service.Service): if not domain_id in self._instances: instance = Instance(conn, domain_id) self._instances[domain_id] = instance - logging.debug('Found instance: %s', domain_id) + logging.debug(_('Found instance: %s'), domain_id) for key in self._instances.keys(): instance = self._instances[key] diff --git a/nova/crypto.py b/nova/crypto.py index aacc50b17..b8405552d 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -19,10 +19,10 @@ Wrappers around standard crypto data elements. Includes root and intermediate CAs, SSH key_pairs and x509 certificates. - """ import base64 +import gettext import hashlib import logging import os @@ -34,28 +34,59 @@ import utils import M2Crypto -from nova import exception +gettext.install('nova', unicode=1) + +from nova import context +from nova import db from nova import flags FLAGS = flags.FLAGS -flags.DEFINE_string('ca_file', 'cacert.pem', 'Filename of root CA') +flags.DEFINE_string('ca_file', 'cacert.pem', _('Filename of root CA')) +flags.DEFINE_string('key_file', + os.path.join('private', 'cakey.pem'), + _('Filename of private key')) +flags.DEFINE_string('crl_file', 'crl.pem', + _('Filename of root Certificate Revokation List')) flags.DEFINE_string('keys_path', '$state_path/keys', - 'Where we keep our keys') + _('Where we keep our keys')) flags.DEFINE_string('ca_path', '$state_path/CA', - 'Where we keep our root CA') -flags.DEFINE_boolean('use_intermediate_ca', False, - 'Should we use intermediate CAs for each project?') + _('Where we keep our root CA')) +flags.DEFINE_boolean('use_project_ca', False, + _('Should we use a CA for each project?')) +flags.DEFINE_string('user_cert_subject', + '/C=US/ST=California/L=MountainView/O=AnsoLabs/' + 'OU=NovaDev/CN=%s-%s-%s', + _('Subject for certificate for users, ' + '%s for project, user, timestamp')) +flags.DEFINE_string('project_cert_subject', + '/C=US/ST=California/L=MountainView/O=AnsoLabs/' + 'OU=NovaDev/CN=project-ca-%s-%s', + _('Subject for certificate for projects, ' + '%s for project, timestamp')) +flags.DEFINE_string('vpn_cert_subject', + '/C=US/ST=California/L=MountainView/O=AnsoLabs/' + 'OU=NovaDev/CN=project-vpn-%s-%s', + _('Subject for certificate for vpns, ' + '%s for project, timestamp')) -def ca_path(project_id): - if project_id: - return "%s/INTER/%s/cacert.pem" % (FLAGS.ca_path, project_id) - return "%s/cacert.pem" % (FLAGS.ca_path) +def ca_folder(project_id=None): + if FLAGS.use_project_ca and project_id: + return os.path.join(FLAGS.ca_path, 'projects', project_id) + return FLAGS.ca_path + + +def ca_path(project_id=None): + return os.path.join(ca_folder(project_id), FLAGS.ca_file) + + +def key_path(project_id=None): + return os.path.join(ca_folder(project_id), FLAGS.key_file) def fetch_ca(project_id=None, chain=True): - if not FLAGS.use_intermediate_ca: + if not FLAGS.use_project_ca: project_id = None buffer = "" if project_id: @@ -92,8 +123,8 @@ def generate_key_pair(bits=1024): def ssl_pub_to_ssh_pub(ssl_public_key, name='root', suffix='nova'): - pub_key_buffer = M2Crypto.BIO.MemoryBuffer(ssl_public_key) - rsa_key = M2Crypto.RSA.load_pub_key_bio(pub_key_buffer) + buf = M2Crypto.BIO.MemoryBuffer(ssl_public_key) + rsa_key = M2Crypto.RSA.load_pub_key_bio(buf) e, n = rsa_key.pub() key_type = 'ssh-rsa' @@ -106,53 +137,134 @@ def ssl_pub_to_ssh_pub(ssl_public_key, name='root', suffix='nova'): return '%s %s %s@%s\n' % (key_type, b64_blob, name, suffix) -def generate_x509_cert(subject, bits=1024): +def revoke_cert(project_id, file_name): + """Revoke a cert by file name""" + start = os.getcwd() + os.chdir(ca_folder(project_id)) + # NOTE(vish): potential race condition here + utils.execute("openssl ca -config ./openssl.cnf -revoke '%s'" % file_name) + utils.execute("openssl ca -gencrl -config ./openssl.cnf -out '%s'" % + FLAGS.crl_file) + os.chdir(start) + + +def revoke_certs_by_user(user_id): + """Revoke all user certs""" + admin = context.get_admin_context() + for cert in db.certificate_get_all_by_user(admin, user_id): + revoke_cert(cert['project_id'], cert['file_name']) + + +def revoke_certs_by_project(project_id): + """Revoke all project certs""" + # NOTE(vish): This is somewhat useless because we can just shut down + # the vpn. + admin = context.get_admin_context() + for cert in db.certificate_get_all_by_project(admin, project_id): + revoke_cert(cert['project_id'], cert['file_name']) + + +def revoke_certs_by_user_and_project(user_id, project_id): + """Revoke certs for user in project""" + admin = context.get_admin_context() + for cert in db.certificate_get_all_by_user(admin, user_id, project_id): + revoke_cert(cert['project_id'], cert['file_name']) + + +def _project_cert_subject(project_id): + """Helper to generate user cert subject""" + return FLAGS.project_cert_subject % (project_id, utils.isotime()) + + +def _vpn_cert_subject(project_id): + """Helper to generate user cert subject""" + return FLAGS.vpn_cert_subject % (project_id, utils.isotime()) + + +def _user_cert_subject(user_id, project_id): + """Helper to generate user cert subject""" + return FLAGS.user_cert_subject % (project_id, user_id, utils.isotime()) + + +def generate_x509_cert(user_id, project_id, bits=1024): + """Generate and sign a cert for user in project""" + subject = _user_cert_subject(user_id, project_id) tmpdir = tempfile.mkdtemp() keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key')) csrfile = os.path.join(tmpdir, 'temp.csr') - logging.debug("openssl genrsa -out %s %s" % (keyfile, bits)) - utils.runthis("Generating private key: %s", - "openssl genrsa -out %s %s" % (keyfile, bits)) - utils.runthis("Generating CSR: %s", - "openssl req -new -key %s -out %s -batch -subj %s" % + utils.execute("openssl genrsa -out %s %s" % (keyfile, bits)) + utils.execute("openssl req -new -key %s -out %s -batch -subj %s" % (keyfile, csrfile, subject)) private_key = open(keyfile).read() csr = open(csrfile).read() shutil.rmtree(tmpdir) - return (private_key, csr) + (serial, signed_csr) = sign_csr(csr, project_id) + fname = os.path.join(ca_folder(project_id), "newcerts/%s.pem" % serial) + cert = {'user_id': user_id, + 'project_id': project_id, + 'file_name': fname} + db.certificate_create(context.get_admin_context(), cert) + return (private_key, signed_csr) -def sign_csr(csr_text, intermediate=None): - if not FLAGS.use_intermediate_ca: - intermediate = None - if not intermediate: - return _sign_csr(csr_text, FLAGS.ca_path) - user_ca = "%s/INTER/%s" % (FLAGS.ca_path, intermediate) - if not os.path.exists(user_ca): +def _ensure_project_folder(project_id): + if not os.path.exists(ca_path(project_id)): start = os.getcwd() - os.chdir(FLAGS.ca_path) - utils.runthis("Generating intermediate CA: %s", - "sh geninter.sh %s" % (intermediate)) + os.chdir(ca_folder()) + utils.execute("sh geninter.sh %s %s" % + (project_id, _project_cert_subject(project_id))) os.chdir(start) - return _sign_csr(csr_text, user_ca) + + +def generate_vpn_files(project_id): + project_folder = ca_folder(project_id) + csr_fn = os.path.join(project_folder, "server.csr") + crt_fn = os.path.join(project_folder, "server.crt") + + if os.path.exists(crt_fn): + return + _ensure_project_folder(project_id) + start = os.getcwd() + os.chdir(ca_folder()) + # TODO(vish): the shell scripts could all be done in python + utils.execute("sh genvpn.sh %s %s" % + (project_id, _vpn_cert_subject(project_id))) + with open(csr_fn, "r") as csrfile: + csr_text = csrfile.read() + (serial, signed_csr) = sign_csr(csr_text, project_id) + with open(crt_fn, "w") as crtfile: + crtfile.write(signed_csr) + os.chdir(start) + + +def sign_csr(csr_text, project_id=None): + if not FLAGS.use_project_ca: + project_id = None + if not project_id: + return _sign_csr(csr_text, ca_folder()) + _ensure_project_folder(project_id) + project_folder = ca_folder(project_id) + return _sign_csr(csr_text, ca_folder(project_id)) def _sign_csr(csr_text, ca_folder): tmpfolder = tempfile.mkdtemp() - csrfile = open("%s/inbound.csr" % (tmpfolder), "w") + inbound = os.path.join(tmpfolder, "inbound.csr") + outbound = os.path.join(tmpfolder, "outbound.csr") + csrfile = open(inbound, "w") csrfile.write(csr_text) csrfile.close() - logging.debug("Flags path: %s" % ca_folder) + logging.debug(_("Flags path: %s") % ca_folder) start = os.getcwd() # Change working dir to CA os.chdir(ca_folder) - utils.runthis("Signing cert: %s", - "openssl ca -batch -out %s/outbound.crt " - "-config ./openssl.cnf -infiles %s/inbound.csr" % - (tmpfolder, tmpfolder)) + utils.execute("openssl ca -batch -out %s -config " + "./openssl.cnf -infiles %s" % (outbound, inbound)) + out, _err = utils.execute("openssl x509 -in %s -serial -noout" % outbound) + serial = out.rpartition("=")[2] os.chdir(start) - with open("%s/outbound.crt" % (tmpfolder), "r") as crtfile: - return crtfile.read() + with open(outbound, "r") as crtfile: + return (serial, crtfile.read()) def mkreq(bits, subject="foo", ca=0): @@ -160,8 +272,7 @@ def mkreq(bits, subject="foo", ca=0): req = M2Crypto.X509.Request() rsa = M2Crypto.RSA.gen_key(bits, 65537, callback=lambda: None) pk.assign_rsa(rsa) - # Should not be freed here - rsa = None + rsa = None # should not be freed here req.set_pubkey(pk) req.set_subject(subject) req.sign(pk, 'sha512') @@ -225,7 +336,6 @@ def mkcacert(subject='nova', years=1): # IN THE SOFTWARE. # http://code.google.com/p/boto - def compute_md5(fp): """ :type fp: file diff --git a/nova/db/api.py b/nova/db/api.py index 71e8151e7..e0f48d07e 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -130,6 +130,45 @@ def service_update(context, service_id, values): ################### +def certificate_create(context, values): + """Create a certificate from the values dictionary.""" + return IMPL.certificate_create(context, values) + + +def certificate_destroy(context, certificate_id): + """Destroy the certificate or raise if it does not exist.""" + return IMPL.certificate_destroy(context, certificate_id) + + +def certificate_get_all_by_project(context, project_id): + """Get all certificates for a project.""" + return IMPL.certificate_get_all_by_project(context, project_id) + + +def certificate_get_all_by_user(context, user_id): + """Get all certificates for a user.""" + return IMPL.certificate_get_all_by_user(context, user_id) + + +def certificate_get_all_by_user_and_project(context, user_id, project_id): + """Get all certificates for a user and project.""" + return IMPL.certificate_get_all_by_user_and_project(context, + user_id, + project_id) + + +def certificate_update(context, certificate_id, values): + """Set the given properties on an certificate and update it. + + Raises NotFound if service does not exist. + + """ + return IMPL.service_update(context, certificate_id, values) + + +################### + + def floating_ip_allocate_address(context, host, project_id): """Allocate free floating ip and return the address. @@ -308,6 +347,11 @@ def instance_get_floating_address(context, instance_id): return IMPL.instance_get_floating_address(context, instance_id) +def instance_get_project_vpn(context, project_id): + """Get a vpn instance by project or return None.""" + return IMPL.instance_get_project_vpn(context, project_id) + + def instance_get_by_internal_id(context, internal_id): """Get an instance by internal id.""" return IMPL.instance_get_by_internal_id(context, internal_id) @@ -364,6 +408,11 @@ def instance_get_disk_sum_by_host_and_project(context, hostname, proj_id): proj_id) +def instance_action_create(context, values): + """Create an instance action from the values dictionary.""" + return IMPL.instance_action_create(context, values) + + ################### @@ -498,12 +547,14 @@ def network_update(context, network_id, values): ################### -def project_get_network(context, project_id): +def project_get_network(context, project_id, associate=True): """Return the network associated with the project. - Raises NotFound if no such network can be found. + If associate is true, it will attempt to associate a new + network if one is not found, otherwise it returns None. """ + return IMPL.project_get_network(context, project_id) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index e4792fe23..64af223c4 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -41,7 +41,7 @@ FLAGS = flags.FLAGS def is_admin_context(context): """Indicates if the request context is an administrator.""" if not context: - warnings.warn('Use of empty request context is deprecated', + warnings.warn(_('Use of empty request context is deprecated'), DeprecationWarning) raise Exception('die') return context.is_admin @@ -130,7 +130,7 @@ def service_get(context, service_id, session=None): first() if not result: - raise exception.NotFound('No service for id %s' % service_id) + raise exception.NotFound(_('No service for id %s') % service_id) return result @@ -227,7 +227,7 @@ def service_get_by_args(context, host, binary): filter_by(deleted=can_read_deleted(context)).\ first() if not result: - raise exception.NotFound('No service for %s, %s' % (host, binary)) + raise exception.NotFound(_('No service for %s, %s') % (host, binary)) return result @@ -252,6 +252,84 @@ def service_update(context, service_id, values): ################### +@require_admin_context +def certificate_get(context, certificate_id, session=None): + if not session: + session = get_session() + + result = session.query(models.Certificate).\ + filter_by(id=certificate_id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() + + if not result: + raise exception.NotFound('No certificate for id %s' % certificate_id) + + return result + + +@require_admin_context +def certificate_create(context, values): + certificate_ref = models.Certificate() + for (key, value) in values.iteritems(): + certificate_ref[key] = value + certificate_ref.save() + return certificate_ref + + +@require_admin_context +def certificate_destroy(context, certificate_id): + session = get_session() + with session.begin(): + certificate_ref = certificate_get(context, + certificate_id, + session=session) + certificate_ref.delete(session=session) + + +@require_admin_context +def certificate_get_all_by_project(context, project_id): + session = get_session() + return session.query(models.Certificate).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + all() + + +@require_admin_context +def certificate_get_all_by_user(context, user_id): + session = get_session() + return session.query(models.Certificate).\ + filter_by(user_id=user_id).\ + filter_by(deleted=False).\ + all() + + +@require_admin_context +def certificate_get_all_by_user_and_project(_context, user_id, project_id): + session = get_session() + return session.query(models.Certificate).\ + filter_by(user_id=user_id).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + all() + + +@require_admin_context +def certificate_update(context, certificate_id, values): + session = get_session() + with session.begin(): + certificate_ref = certificate_get(context, + certificate_id, + session=session) + for (key, value) in values.iteritems(): + certificate_ref[key] = value + certificate_ref.save(session=session) + + +################### + + @require_context def floating_ip_allocate_address(context, host, project_id): authorize_project_context(context, project_id) @@ -385,6 +463,7 @@ def floating_ip_get_by_address(context, address, session=None): session = get_session() result = session.query(models.FloatingIp).\ + options(joinedload_all('fixed_ip.network')).\ filter_by(address=address).\ filter_by(deleted=can_read_deleted(context)).\ first() @@ -501,7 +580,7 @@ def fixed_ip_get_by_address(context, address, session=None): options(joinedload('instance')).\ first() if not result: - raise exception.NotFound('No floating ip for address %s' % address) + raise exception.NotFound(_('No floating ip for address %s') % address) if is_user_context(context): authorize_project_context(context, result.instance.project_id) @@ -538,6 +617,8 @@ def fixed_ip_update(context, address, values): #TODO(gundlach): instance_create and volume_create are nearly identical #and should be refactored. I expect there are other copy-and-paste #functions between the two of them as well. + + @require_context def instance_create(context, values): """Create a new Instance record in the database. @@ -553,7 +634,7 @@ def instance_create(context, values): with session.begin(): while instance_ref.internal_id == None: # Instances have integer internal ids. - internal_id = random.randint(0, 2 ** 32 - 1) + internal_id = random.randint(0, 2 ** 31 - 1) if not instance_internal_id_exists(context, internal_id, session=session): instance_ref.internal_id = internal_id @@ -589,19 +670,23 @@ def instance_get(context, instance_id, session=None): if is_admin_context(context): result = session.query(models.Instance).\ + options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload('security_groups')).\ + options(joinedload('volumes')).\ filter_by(id=instance_id).\ filter_by(deleted=can_read_deleted(context)).\ first() elif is_user_context(context): result = session.query(models.Instance).\ + options(joinedload_all('fixed_ip.floating_ips')).\ options(joinedload('security_groups')).\ + options(joinedload('volumes')).\ filter_by(project_id=context.project_id).\ filter_by(id=instance_id).\ filter_by(deleted=False).\ first() if not result: - raise exception.NotFound('No instance for id %s' % instance_id) + raise exception.NotFound(_('No instance for id %s') % instance_id) return result @@ -661,6 +746,18 @@ def instance_get_all_by_reservation(context, reservation_id): all() +@require_admin_context +def instance_get_project_vpn(context, project_id): + session = get_session() + return session.query(models.Instance).\ + options(joinedload_all('fixed_ip.floating_ips')).\ + options(joinedload('security_groups')).\ + filter_by(project_id=project_id).\ + filter_by(image_id=FLAGS.vpn_image_id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() + + @require_context def instance_get_by_internal_id(context, internal_id): session = get_session() @@ -679,7 +776,7 @@ def instance_get_by_internal_id(context, internal_id): filter_by(deleted=False).\ first() if not result: - raise exception.NotFound('Instance %s not found' % (internal_id)) + raise exception.NotFound(_('Instance %s not found') % (internal_id)) return result @@ -742,6 +839,7 @@ def instance_update(context, instance_id, values): instance_ref = instance_get(context, instance_id, session=session) instance_ref.update(values) instance_ref.save(session=session) + return instance_ref @require_context @@ -802,6 +900,17 @@ def instance_get_memory_sum_by_host_and_project(context, hostname, proj_id): def instance_get_disk_sum_by_host_and_project(context, hostname, proj_id): return _instance_get_sum_by_host_and_project(context, 'local_gb', hostname, proj_id) +@require_context +def instance_action_create(context, values): + """Create an instance action from the values dictionary.""" + action_ref = models.InstanceActions() + action_ref.update(values) + + session = get_session() + with session.begin(): + action_ref.save(session=session) + return action_ref + ################### @@ -846,7 +955,7 @@ def key_pair_get(context, user_id, name, session=None): filter_by(deleted=can_read_deleted(context)).\ first() if not result: - raise exception.NotFound('no keypair for user %s, name %s' % + raise exception.NotFound(_('no keypair for user %s, name %s') % (user_id, name)) return result @@ -961,7 +1070,7 @@ def network_get(context, network_id, session=None): filter_by(deleted=False).\ first() if not result: - raise exception.NotFound('No network for id %s' % network_id) + raise exception.NotFound(_('No network for id %s') % network_id) return result @@ -969,6 +1078,8 @@ def network_get(context, network_id, session=None): # NOTE(vish): pylint complains because of the long method name, but # it fits with the names of the rest of the methods # pylint: disable-msg=C0103 + + @require_admin_context def network_get_associated_fixed_ips(context, network_id): session = get_session() @@ -989,7 +1100,7 @@ def network_get_by_bridge(context, bridge): first() if not result: - raise exception.NotFound('No network for bridge %s' % bridge) + raise exception.NotFound(_('No network for bridge %s') % bridge) return result @@ -1003,7 +1114,7 @@ def network_get_by_instance(_context, instance_id): filter_by(deleted=False).\ first() if not rv: - raise exception.NotFound('No network for instance %s' % instance_id) + raise exception.NotFound(_('No network for instance %s') % instance_id) return rv @@ -1017,7 +1128,7 @@ def network_set_host(context, network_id, host_id): with_lockmode('update').\ first() if not network_ref: - raise exception.NotFound('No network for id %s' % network_id) + raise exception.NotFound(_('No network for id %s') % network_id) # NOTE(vish): if with_lockmode isn't supported, as in sqlite, # then this has concurrency issues @@ -1041,24 +1152,26 @@ def network_update(context, network_id, values): @require_context -def project_get_network(context, project_id): +def project_get_network(context, project_id, associate=True): session = get_session() - rv = session.query(models.Network).\ - filter_by(project_id=project_id).\ - filter_by(deleted=False).\ - first() - if not rv: + result = session.query(models.Network).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + first() + if not result: + if not associate: + return None try: return network_associate(context, project_id) except IntegrityError: # NOTE(vish): We hit this if there is a race and two # processes are attempting to allocate the # network at the same time - rv = session.query(models.Network).\ - filter_by(project_id=project_id).\ - filter_by(deleted=False).\ - first() - return rv + result = session.query(models.Network).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + first() + return result ################### @@ -1118,21 +1231,24 @@ def iscsi_target_create_safe(context, values): ################### +@require_admin_context def auth_destroy_token(_context, token): session = get_session() session.delete(token) +@require_admin_context def auth_get_token(_context, token_hash): session = get_session() tk = session.query(models.AuthToken).\ filter_by(token_hash=token_hash).\ first() if not tk: - raise exception.NotFound('Token %s does not exist' % token_hash) + raise exception.NotFound(_('Token %s does not exist') % token_hash) return tk +@require_admin_context def auth_create_token(_context, token): tk = models.AuthToken() tk.update(token) @@ -1153,7 +1269,7 @@ def quota_get(context, project_id, session=None): filter_by(deleted=can_read_deleted(context)).\ first() if not result: - raise exception.NotFound('No quota for project_id %s' % project_id) + raise exception.NotFound(_('No quota for project_id %s') % project_id) return result @@ -1308,7 +1424,7 @@ def volume_get(context, volume_id, session=None): filter_by(deleted=False).\ first() if not result: - raise exception.NotFound('No volume for id %s' % volume_id) + raise exception.NotFound(_('No volume for id %s') % volume_id) return result @@ -1364,7 +1480,7 @@ def volume_get_by_ec2_id(context, ec2_id): raise exception.NotAuthorized() if not result: - raise exception.NotFound('Volume %s not found' % ec2_id) + raise exception.NotFound(_('Volume %s not found') % ec2_id) return result @@ -1388,7 +1504,7 @@ def volume_get_instance(context, volume_id): options(joinedload('instance')).\ first() if not result: - raise exception.NotFound('Volume %s not found' % ec2_id) + raise exception.NotFound(_('Volume %s not found') % ec2_id) return result.instance @@ -1400,7 +1516,7 @@ def volume_get_shelf_and_blade(context, volume_id): filter_by(volume_id=volume_id).\ first() if not result: - raise exception.NotFound('No export device found for volume %s' % + raise exception.NotFound(_('No export device found for volume %s') % volume_id) return (result.shelf_id, result.blade_id) @@ -1413,7 +1529,7 @@ def volume_get_iscsi_target_num(context, volume_id): filter_by(volume_id=volume_id).\ first() if not result: - raise exception.NotFound('No target id found for volume %s' % + raise exception.NotFound(_('No target id found for volume %s') % volume_id) return result.target_num @@ -1458,7 +1574,7 @@ def security_group_get(context, security_group_id, session=None): options(joinedload_all('rules')).\ first() if not result: - raise exception.NotFound("No secuity group with id %s" % + raise exception.NotFound(_("No security group with id %s") % security_group_id) return result @@ -1475,7 +1591,7 @@ def security_group_get_by_name(context, project_id, group_name): first() if not result: raise exception.NotFound( - 'No security group named %s for project: %s' \ + _('No security group named %s for project: %s') % (group_name, project_id)) return result @@ -1563,7 +1679,7 @@ def security_group_rule_get(context, security_group_rule_id, session=None): filter_by(id=security_group_rule_id).\ first() if not result: - raise exception.NotFound("No secuity group rule with id %s" % + raise exception.NotFound(_("No secuity group rule with id %s") % security_group_rule_id) return result @@ -1599,7 +1715,7 @@ def user_get(context, id, session=None): first() if not result: - raise exception.NotFound('No user for id %s' % id) + raise exception.NotFound(_('No user for id %s') % id) return result @@ -1615,7 +1731,7 @@ def user_get_by_access_key(context, access_key, session=None): first() if not result: - raise exception.NotFound('No user for access key %s' % access_key) + raise exception.NotFound(_('No user for access key %s') % access_key) return result @@ -1677,7 +1793,7 @@ def project_get(context, id, session=None): first() if not result: - raise exception.NotFound("No project with id %s" % id) + raise exception.NotFound(_("No project with id %s") % id) return result diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index c3e566c6a..a173f25db 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -22,7 +22,7 @@ SQLAlchemy models for nova data. import datetime from sqlalchemy.orm import relationship, backref, object_mapper -from sqlalchemy import Column, Integer, String, schema +from sqlalchemy import Column, Integer, Float, String, schema from sqlalchemy import ForeignKey, DateTime, Boolean, Text from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declarative_base @@ -171,6 +171,16 @@ class Service(BASE, NovaBase): disabled = Column(Boolean, default=False) +class Certificate(BASE, NovaBase): + """Represents a an x509 certificate""" + __tablename__ = 'certificates' + id = Column(Integer, primary_key=True) + + user_id = Column(String(255)) + project_id = Column(String(255)) + file_name = Column(String(255)) + + class Instance(BASE, NovaBase): """Represents a guest vm.""" __tablename__ = 'instances' @@ -250,6 +260,31 @@ class Instance(BASE, NovaBase): # 'shutdown', 'shutoff', 'crashed']) +class InstanceDiagnostics(BASE, NovaBase): + """Represents a guest VM's diagnostics""" + __tablename__ = "instance_diagnostics" + id = Column(Integer, primary_key=True) + instance_id = Column(Integer, ForeignKey('instances.id')) + + memory_available = Column(Float) + memory_free = Column(Float) + cpu_load = Column(Float) + disk_read = Column(Float) + disk_write = Column(Float) + net_tx = Column(Float) + net_rx = Column(Float) + + +class InstanceActions(BASE, NovaBase): + """Represents a guest VM's actions and results""" + __tablename__ = "instance_actions" + id = Column(Integer, primary_key=True) + instance_id = Column(Integer, ForeignKey('instances.id')) + + action = Column(String(255)) + error = Column(Text) + + class Volume(BASE, NovaBase): """Represents a block storage device that can be attached to a vm.""" __tablename__ = 'volumes' @@ -550,10 +585,6 @@ def register_models(): it will never need to be called explicitly elsewhere. """ from sqlalchemy import create_engine - #models = (Service, Instance, Volume, ExportDevice, IscsiTarget, FixedIp, - # FloatingIp, Network, SecurityGroup, - # SecurityGroupIngressRule, SecurityGroupInstanceAssociation, - # AuthToken, User, Project) # , Image, Host models = (Service, Instance, Volume, ExportDevice, IscsiTarget, FixedIp, FloatingIp, Network, SecurityGroup, SecurityGroupIngressRule, SecurityGroupInstanceAssociation, diff --git a/nova/exception.py b/nova/exception.py index 6d6c37338..277033e0f 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -27,23 +27,26 @@ import traceback class ProcessExecutionError(IOError): + def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, description=None): if description is None: - description = "Unexpected error while running command." + description = _("Unexpected error while running command.") if exit_code is None: exit_code = '-' - message = "%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" % ( - description, cmd, exit_code, stdout, stderr) + message = _("%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r")\ + % (description, cmd, exit_code, stdout, stderr) IOError.__init__(self, message) class Error(Exception): + def __init__(self, message=None): super(Error, self).__init__(message) class ApiError(Error): + def __init__(self, message='Unknown', code='Unknown'): self.message = message self.code = code @@ -81,7 +84,7 @@ def wrap_exception(f): except Exception, e: if not isinstance(e, Error): #exc_type, exc_value, exc_traceback = sys.exc_info() - logging.exception('Uncaught exception') + logging.exception(_('Uncaught exception')) #logging.error(traceback.extract_stack(exc_traceback)) raise Error(str(e)) raise diff --git a/nova/fakememcache.py b/nova/fakememcache.py new file mode 100644 index 000000000..67f46dbdc --- /dev/null +++ b/nova/fakememcache.py @@ -0,0 +1,59 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Super simple fake memcache client.""" + +import utils + + +class Client(object): + """Replicates a tiny subset of memcached client interface.""" + + def __init__(self, *args, **kwargs): + """Ignores the passed in args""" + self.cache = {} + + def get(self, key): + """Retrieves the value for a key or None.""" + (timeout, value) = self.cache.get(key, (0, None)) + if timeout == 0 or utils.utcnow_ts() < timeout: + return value + return None + + def set(self, key, value, time=0, min_compress_len=0): + """Sets the value for a key.""" + timeout = 0 + if time != 0: + timeout = utils.utcnow_ts() + time + self.cache[key] = (timeout, value) + return True + + def add(self, key, value, time=0, min_compress_len=0): + """Sets the value for a key if it doesn't exist.""" + if not self.get(key) is None: + return False + return self.set(key, value, time, min_compress_len) + + def incr(self, key, delta=1): + """Increments the value for a key.""" + value = self.get(key) + if value is None: + return None + new_value = int(value) + delta + self.cache[key] = (self.cache[key][0], str(new_value)) + return new_value diff --git a/nova/fakerabbit.py b/nova/fakerabbit.py index c64617931..79d8b894d 100644 --- a/nova/fakerabbit.py +++ b/nova/fakerabbit.py @@ -25,6 +25,10 @@ from carrot.backends import base from eventlet import greenthread +EXCHANGES = {} +QUEUES = {} + + class Message(base.BaseMessage): pass @@ -37,12 +41,12 @@ class Exchange(object): self._routes = {} def publish(self, message, routing_key=None): - logging.debug('(%s) publish (key: %s) %s', + logging.debug(_('(%s) publish (key: %s) %s'), self.name, routing_key, message) routing_key = routing_key.split('.')[0] if routing_key in self._routes: for f in self._routes[routing_key]: - logging.debug('Publishing to route %s', f) + logging.debug(_('Publishing to route %s'), f) f(message, routing_key=routing_key) def bind(self, callback, routing_key): @@ -68,81 +72,63 @@ class Queue(object): return self._queue.get() -class Backend(object): - """ Singleton backend for testing """ - class __impl(base.BaseBackend): - def __init__(self, *args, **kwargs): - #super(__impl, self).__init__(*args, **kwargs) - self._exchanges = {} - self._queues = {} - - def _reset_all(self): - self._exchanges = {} - self._queues = {} - - def queue_declare(self, queue, **kwargs): - if queue not in self._queues: - logging.debug('Declaring queue %s', queue) - self._queues[queue] = Queue(queue) - - def exchange_declare(self, exchange, type, *args, **kwargs): - if exchange not in self._exchanges: - logging.debug('Declaring exchange %s', exchange) - self._exchanges[exchange] = Exchange(exchange, type) - - def queue_bind(self, queue, exchange, routing_key, **kwargs): - logging.debug('Binding %s to %s with key %s', - queue, exchange, routing_key) - self._exchanges[exchange].bind(self._queues[queue].push, - routing_key) - - def declare_consumer(self, queue, callback, *args, **kwargs): - self.current_queue = queue - self.current_callback = callback - - def consume(self, *args, **kwargs): - while True: - item = self.get(self.current_queue) - if item: - self.current_callback(item) - raise StopIteration() - greenthread.sleep(0) - - def get(self, queue, no_ack=False): - if not queue in self._queues or not self._queues[queue].size(): - return None - (message_data, content_type, content_encoding) = \ - self._queues[queue].pop() - message = Message(backend=self, body=message_data, - content_type=content_type, - content_encoding=content_encoding) - message.result = True - logging.debug('Getting from %s: %s', queue, message) - return message - - def prepare_message(self, message_data, delivery_mode, - content_type, content_encoding, **kwargs): - """Prepare message for sending.""" - return (message_data, content_type, content_encoding) - - def publish(self, message, exchange, routing_key, **kwargs): - if exchange in self._exchanges: - self._exchanges[exchange].publish( - message, routing_key=routing_key) - - __instance = None - - def __init__(self, *args, **kwargs): - if Backend.__instance is None: - Backend.__instance = Backend.__impl(*args, **kwargs) - self.__dict__['_Backend__instance'] = Backend.__instance - - def __getattr__(self, attr): - return getattr(self.__instance, attr) - - def __setattr__(self, attr, value): - return setattr(self.__instance, attr, value) +class Backend(base.BaseBackend): + def queue_declare(self, queue, **kwargs): + global QUEUES + if queue not in QUEUES: + logging.debug(_('Declaring queue %s'), queue) + QUEUES[queue] = Queue(queue) + + def exchange_declare(self, exchange, type, *args, **kwargs): + global EXCHANGES + if exchange not in EXCHANGES: + logging.debug(_('Declaring exchange %s'), exchange) + EXCHANGES[exchange] = Exchange(exchange, type) + + def queue_bind(self, queue, exchange, routing_key, **kwargs): + global EXCHANGES + global QUEUES + logging.debug(_('Binding %s to %s with key %s'), + queue, exchange, routing_key) + EXCHANGES[exchange].bind(QUEUES[queue].push, routing_key) + + def declare_consumer(self, queue, callback, *args, **kwargs): + self.current_queue = queue + self.current_callback = callback + + def consume(self, limit=None): + while True: + item = self.get(self.current_queue) + if item: + self.current_callback(item) + raise StopIteration() + greenthread.sleep(0) + + def get(self, queue, no_ack=False): + global QUEUES + if not queue in QUEUES or not QUEUES[queue].size(): + return None + (message_data, content_type, content_encoding) = QUEUES[queue].pop() + message = Message(backend=self, body=message_data, + content_type=content_type, + content_encoding=content_encoding) + message.result = True + logging.debug(_('Getting from %s: %s'), queue, message) + return message + + def prepare_message(self, message_data, delivery_mode, + content_type, content_encoding, **kwargs): + """Prepare message for sending.""" + return (message_data, content_type, content_encoding) + + def publish(self, message, exchange, routing_key, **kwargs): + global EXCHANGES + if exchange in EXCHANGES: + EXCHANGES[exchange].publish(message, routing_key=routing_key) def reset_all(): - Backend()._reset_all() + global EXCHANGES + global QUEUES + EXCHANGES = {} + QUEUES = {} diff --git a/nova/flags.py b/nova/flags.py index 1f94feb08..76a98d35a 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -29,6 +29,8 @@ import sys import gflags +from nova import utils + class FlagValues(gflags.FlagValues): """Extension of gflags.FlagValues that allows undefined and runtime flags. @@ -159,6 +161,7 @@ class StrWrapper(object): return str(val) raise KeyError(name) + FLAGS = FlagValues() gflags.FLAGS = FLAGS gflags.DEFINE_flag(gflags.HelpFlag(), FLAGS) @@ -183,6 +186,12 @@ DEFINE_list = _wrapper(gflags.DEFINE_list) DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist) DEFINE_multistring = _wrapper(gflags.DEFINE_multistring) DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int) +DEFINE_flag = _wrapper(gflags.DEFINE_flag) + + +HelpFlag = gflags.HelpFlag +HelpshortFlag = gflags.HelpshortFlag +HelpXMLFlag = gflags.HelpXMLFlag def DECLARE(name, module_string, flag_values=FLAGS): @@ -204,7 +213,8 @@ DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake') DEFINE_string('aws_access_key_id', 'admin', 'AWS Access ID') DEFINE_string('aws_secret_access_key', 'admin', 'AWS Access Key') DEFINE_integer('s3_port', 3333, 's3 port') -DEFINE_string('s3_host', '127.0.0.1', 's3 host') +DEFINE_string('s3_host', utils.get_my_ip(), 's3 host (for infrastructure)') +DEFINE_string('s3_dmz', utils.get_my_ip(), 's3 dmz ip (for instances)') DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on') DEFINE_string('scheduler_topic', 'scheduler', 'the topic scheduler nodes listen on') @@ -223,22 +233,24 @@ DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host') DEFINE_integer('rabbit_retry_interval', 10, 'rabbit connection retry interval') DEFINE_integer('rabbit_max_retries', 12, 'rabbit connection attempts') DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to') -DEFINE_string('ec2_url', 'http://127.0.0.1:8773/services/Cloud', - 'Url to ec2 api server') +DEFINE_string('ec2_prefix', 'http', 'prefix for ec2') +DEFINE_string('cc_host', utils.get_my_ip(), 'ip of api server') +DEFINE_string('cc_dmz', utils.get_my_ip(), 'internal ip of api server') +DEFINE_integer('cc_port', 8773, 'cloud controller port') +DEFINE_string('ec2_suffix', '/services/Cloud', 'suffix for ec2') DEFINE_string('default_image', 'ami-11111', 'default image to use, testing only') -DEFINE_string('default_kernel', 'aki-11111', - 'default kernel to use, testing only') -DEFINE_string('default_ramdisk', 'ari-11111', - 'default ramdisk to use, testing only') DEFINE_string('default_instance_type', 'm1.small', 'default instance type to use, testing only') +DEFINE_string('null_kernel', 'nokernel', + 'kernel image that indicates not to use a kernel,' + ' but to use a raw disk image instead') -DEFINE_string('vpn_image_id', 'ami-CLOUDPIPE', 'AMI for cloudpipe vpn server') +DEFINE_string('vpn_image_id', 'ami-cloudpipe', 'AMI for cloudpipe vpn server') DEFINE_string('vpn_key_suffix', - '-key', - 'Suffix to add to project name for vpn key') + '-vpn', + 'Suffix to add to project name for vpn key and secgroups') DEFINE_integer('auth_token_ttl', 3600, 'Seconds for auth tokens to linger') @@ -259,7 +271,7 @@ DEFINE_string('scheduler_manager', 'nova.scheduler.manager.SchedulerManager', 'Manager for scheduler') # The service to use for image search and retrieval -DEFINE_string('image_service', 'nova.image.local.LocalImageService', +DEFINE_string('image_service', 'nova.image.s3.S3ImageService', 'The service to use for retrieving and searching for images.') DEFINE_string('host', socket.gethostname(), diff --git a/nova/image/glance.py b/nova/image/glance.py index 1ca6cf2eb..cb3936df1 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -77,8 +77,8 @@ class ParallaxClient(object): data = json.loads(res.read())['images'] return data else: - logging.warn("Parallax returned HTTP error %d from " - "request for /images", res.status_int) + logging.warn(_("Parallax returned HTTP error %d from " + "request for /images"), res.status_int) return [] finally: c.close() @@ -96,8 +96,8 @@ class ParallaxClient(object): data = json.loads(res.read())['images'] return data else: - logging.warn("Parallax returned HTTP error %d from " - "request for /images/detail", res.status_int) + logging.warn(_("Parallax returned HTTP error %d from " + "request for /images/detail"), res.status_int) return [] finally: c.close() diff --git a/nova/image/local.py b/nova/image/local.py index 9b0cdcc50..b44593221 100644 --- a/nova/image/local.py +++ b/nova/image/local.py @@ -59,7 +59,7 @@ class LocalImageService(service.BaseImageService): """ Store the image data and return the new image id. """ - id = random.randint(0, 2 ** 32 - 1) + id = random.randint(0, 2 ** 31 - 1) data['id'] = id self.update(context, id, data) return id diff --git a/nova/image/s3.py b/nova/image/s3.py index 0a25161de..7b04aa072 100644 --- a/nova/image/s3.py +++ b/nova/image/s3.py @@ -79,7 +79,8 @@ class S3ImageService(service.BaseImageService): result = self.index(context) result = [i for i in result if i['imageId'] == image_id] if not result: - raise exception.NotFound('Image %s could not be found' % image_id) + raise exception.NotFound(_('Image %s could not be found') + % image_id) image = result[0] return image diff --git a/nova/livemigration_test/SI/testCase_SI.xls b/nova/livemigration_test/SI/testCase_SI.xls Binary files differindex 65cf96fd7..be98b391a 100644 --- a/nova/livemigration_test/SI/testCase_SI.xls +++ b/nova/livemigration_test/SI/testCase_SI.xls diff --git a/nova/livemigration_test/UT/libvirtConnection.test.py b/nova/livemigration_test/UT/libvirtConnection.test.py index 5dfe8702c..0b737e140 100644 --- a/nova/livemigration_test/UT/libvirtConnection.test.py +++ b/nova/livemigration_test/UT/libvirtConnection.test.py @@ -35,7 +35,6 @@ try : from nova import flags from nova import quota from nova import utils - from nova import process from nova.auth import manager from nova.cloudpipe import pipelib from nova import rpc @@ -177,10 +176,11 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): def test03(self): """03: Unexpected exception occurs on finding volume on DB. """ - utils.execute = Mock( side_effect=process.ProcessExecutionError('ERR') ) + utils.execute = Mock( side_effect=exception.ProcessExecutionError('ERR') ) - self.assertRaises(process.ProcessExecutionError, - self.manager.live_migration, + self.assertRaises(exception.ProcessExecutionError, + self.manager._live_migration, + self.ctxt, self.instance1, 'host2') @@ -195,6 +195,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): self.assertRaises(TypeError, self.manager._post_live_migration, + self.ctxt, "dummy string", 'host2') @@ -202,7 +203,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): """05: db.instance_get_fixed_address return None""" db.instance_get_fixed_address = Mock( return_value=None ) - ret = self.manager._post_live_migration(self.instance1, 'host1') + ret = self.manager._post_live_migration(self.ctxt, self.instance1, 'host1') c1 = (ret == None) c2 = (0 <= sys.stderr.buffer.find('fixed_ip is not found')) self.assertEqual(c1 and c2, True) @@ -213,6 +214,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): db.instance_get_fixed_address = Mock( side_effect=exception.NotFound('ERR') ) self.assertRaises(exception.NotFound, self.manager._post_live_migration, + self.ctxt, self.instance1, 'host2') @@ -222,6 +224,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): db.instance_get_fixed_address = Mock( side_effect=TypeError('ERR') ) self.assertRaises(TypeError, self.manager._post_live_migration, + self.ctxt, self.instance1, 'host1') @@ -231,6 +234,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): db.fixed_ip_update = Mock( side_effect=exception.NotFound('ERR') ) self.assertRaises(exception.NotFound, self.manager._post_live_migration, + self.ctxt, self.instance1, 'host1') @@ -239,6 +243,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): db.fixed_ip_update = Mock( side_effect=exception.NotAuthorized('ERR') ) self.assertRaises(exception.NotAuthorized, self.manager._post_live_migration, + self.ctxt, self.instance1, 'host1') @@ -247,6 +252,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): db.fixed_ip_update = Mock( side_effect=TypeError('ERR') ) self.assertRaises(TypeError, self.manager._post_live_migration, + self.ctxt, self.instance1, 'host1') @@ -256,6 +262,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): db.fixed_ip_get_network = Mock( side_effect=exception.NotFound('ERR') ) self.assertRaises(exception.NotFound, self.manager._post_live_migration, + self.ctxt, self.instance1, 'host1') @@ -268,6 +275,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): db.fixed_ip_get_network = Mock( side_effect=TypeError('ERR') ) self.assertRaises(TypeError, self.manager._post_live_migration, + self.ctxt, self.instance1, 'host1') @@ -276,13 +284,14 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): db.network_update = Mock( side_effect=TypeError('ERR') ) self.assertRaises(TypeError, self.manager._post_live_migration, + self.ctxt, self.instance1, 'host1') def test14(self): """14: db.instance_get_floating_address raises NotFound. """ db.instance_get_floating_address = Mock(side_effect=exception.NotFound("ERR")) - ret = self.manager._post_live_migration(self.instance1, 'host1') + ret = self.manager._post_live_migration(self.ctxt, self.instance1, 'host1') c1 = (ret == None) c2 = (0 <= sys.stderr.buffer.find('doesnt have floating_ip')) self.assertEqual(c1 and c2, True) @@ -292,7 +301,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): """15: db.instance_get_floating_address returns None. """ db.instance_get_floating_address = Mock( return_value=None ) - ret = self.manager._post_live_migration(self.instance1, 'host1') + ret = self.manager._post_live_migration(self.ctxt, self.instance1, 'host1') c1 = (ret == None) c2 = (0 <= sys.stderr.buffer.find('floating_ip is not found')) self.assertEqual(c1 and c2, True) @@ -301,7 +310,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): """16: db.instance_get_floating_address raises NotFound. """ db.instance_get_floating_address = Mock(side_effect=exception.NotFound("ERR")) - ret = self.manager._post_live_migration(self.instance1, 'host1') + ret = self.manager._post_live_migration(self.ctxt, self.instance1, 'host1') c1 = (ret == None) c2 = (0 <= sys.stderr.buffer.find('doesnt have floating_ip')) self.assertEqual(c1 and c2, True) @@ -309,7 +318,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): def test17(self): """17: db.instance_get_floating_address raises Unknown exception. """ db.instance_get_floating_address = Mock(side_effect=TypeError("ERR")) - ret = self.manager._post_live_migration(self.instance1, 'host1') + ret = self.manager._post_live_migration(self.ctxt, self.instance1, 'host1') c1 = (ret == None) c2 = (0 <= sys.stderr.buffer.find('Live migration: Unexpected error')) self.assertEqual(c1 and c2, True) @@ -319,7 +328,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): """18: db.floating_ip_get_by_address raises NotFound """ db.floating_ip_get_by_address = Mock(side_effect=exception.NotFound("ERR")) - ret = self.manager._post_live_migration(self.instance1, 'host1') + ret = self.manager._post_live_migration(self.ctxt, self.instance1, 'host1') c1 = (ret == None) c2 = (0 <= sys.stderr.buffer.find('doesnt have floating_ip')) self.assertEqual(c1 and c2, True) @@ -327,7 +336,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): def test19(self): """19: db.floating_ip_get_by_address raises Unknown exception. """ db.floating_ip_get_by_address = Mock(side_effect=TypeError("ERR")) - ret = self.manager._post_live_migration(self.instance1, 'host1') + ret = self.manager._post_live_migration(self.ctxt, self.instance1, 'host1') c1 = (ret == None) c2 = (0 <= sys.stderr.buffer.find('Live migration: Unexpected error')) self.assertEqual(c1 and c2, True) @@ -337,7 +346,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): """20: db.floating_ip_update raises Unknown exception. """ db.floating_ip_update = Mock(side_effect=TypeError("ERR")) - ret = self.manager._post_live_migration(self.instance1, 'host1') + ret = self.manager._post_live_migration(self.ctxt, self.instance1, 'host1') c1 = (ret == None) c2 = (0 <= sys.stderr.buffer.find('Live migration: Unexpected error')) self.assertEqual(c1 and c2, True) @@ -348,6 +357,7 @@ class LibvirtConnectionTestFunctions(unittest.TestCase): db.instance_update = Mock(side_effect=TypeError("ERR")) self.assertRaises(TypeError, self.manager._post_live_migration, + self.ctxt, self.instance1, 'host1') diff --git a/nova/livemigration_test/UT/nova-manage.test.py b/nova/livemigration_test/UT/nova-manage.test.py index f1653d21a..6db15cea0 100644 --- a/nova/livemigration_test/UT/nova-manage.test.py +++ b/nova/livemigration_test/UT/nova-manage.test.py @@ -256,73 +256,35 @@ class NovaManageTestFunctions(unittest.TestCase): self.assertRaises(TypeError, self.instanceCmds.live_migration, 'i-xxx' ) def test11(self): - """11: nova-manage instances live_migration ec2_id host, - where hostname is invalid - """ - db.host_get_by_name = Mock( side_effect=exception.NotFound('ERR') ) - self.assertRaises(exception.NotFound, self.instanceCmds.live_migration, 'i-xxx', 'host1' ) - - def test12(self): - """12: nova-manage instances live_migration ec2_id(invalid id) host""" + """11: nova-manage instances live_migration ec2_id(invalid id) host""" - db.host_get_by_name = Mock(return_value = self.host1) db.instance_get_by_internal_id = Mock( side_effect=exception.NotFound('ERR') ) - - self.assertRaises(exception.NotFound, self.instanceCmds.live_migration, 'i-xxx', 'host1' ) - - def test13(self): - """13: nova-manage instances live_migration ec2_id host, - but instance specifed by ec2 id is not running (state is not power_state.RUNNING) - """ - db.host_get_by_name = Mock(return_value = self.host1) - db.instance_get_by_internal_id = Mock( return_value = self.instance1 ) - try : - self.instanceCmds.live_migration('i-12345', 'host1') - except exception.Invalid, e: - c1 = (0 < e.message.find('is not running') ) - self.assertTrue(c1, True) - return False - - - def test14(self): - """14: nova-manage instances live_migration ec2_id host, - but instance specifed by ec2 id is not running (state_description is not running) - """ - db.host_get_by_name = Mock(return_value = self.host2) - db.instance_get_by_internal_id = Mock( return_value = self.instance1 ) try : - self.instanceCmds.live_migration('i-12345', 'host2') - except exception.Invalid, e: - c1 = (0 < e.message.find('is not running') ) + self.instanceCmds.live_migration('i-xxx', 'host1') + except exception.NotFound, e: + c1 = (0 < str(e.args).find('is not found') ) self.assertTrue(c1, True) return False - def test15(self): - """15: nova-manage instances live_migration ec2_id host, - but instance is running at the same host specifed above, so err should be occured. + def test12(self): + """12: nova-manage instances live_migration ec2_id host + and db.instance_get_by_internal_id raises unexpected exceptioin. """ - db.host_get_by_name = Mock(return_value = self.host1) - db.instance_get_by_internal_id = Mock( return_value = self.instance3 ) - try : - self.instanceCmds.live_migration('i-12345', 'host1') - except exception.Invalid, e: - c1 = ( 0 <= e.message.find('is running now') ) - self.assertTrue(c1, True) - return False - + db.instance_get_by_internal_id = Mock( side_effect=TypeError('ERR') ) + self.assertRaises(TypeError, self.instanceCmds.live_migration, 'i-xxx' ) - def test16(self): - """16: nova-manage instances live_migration ec2_id host, + def test13(self): + """13: nova-manage instances live_migration ec2_id host, rpc.call raises RemoteError because destination doesnt have enough resource. """ db.host_get_by_name = Mock(return_value = self.host1) db.instance_get_by_internal_id = Mock( return_value = self.instance3 ) rpc.call = Mock(return_value = rpc.RemoteError(TypeError, 'val', 'traceback')) self.assertRaises(rpc.RemoteError, self.instanceCmds.live_migration, 'i-xxx', 'host2' ) - - def test17(self): - """17: nova-manage instances live_migration ec2_id host, + + def test14(self): + """14: nova-manage instances live_migration ec2_id host, everything goes well, ang gets success messages. """ db.host_get_by_name = Mock(return_value = self.host1) @@ -332,7 +294,7 @@ class NovaManageTestFunctions(unittest.TestCase): self.instanceCmds.live_migration('i-12345', 'host2') c1 = (0 <= self.stdout.buffer.find('Finished all procedure') ) self.assertEqual( c1, True ) - + def tearDown(self): """common terminating method. """ diff --git a/nova/livemigration_test/UT/schedulerManager.test.py b/nova/livemigration_test/UT/schedulerManager.test.py index a0b76c918..33a38c660 100644 --- a/nova/livemigration_test/UT/schedulerManager.test.py +++ b/nova/livemigration_test/UT/schedulerManager.test.py @@ -55,12 +55,15 @@ class tmpStdout: class SchedulerTestFunctions(unittest.TestCase): + manager = None + # 共通の初期化処理 def setUp(self): """common init method. """ self.host = 'openstack2-api' - self.manager = SchedulerManager(host=self.host) + if self.manager is None: + self.manager = SchedulerManager(host=self.host) self.setTestData() self.setMocks() @@ -72,6 +75,7 @@ class SchedulerTestFunctions(unittest.TestCase): self.host1.__setitem__('vcpus', 5) self.host1.__setitem__('memory_mb', 20480) self.host1.__setitem__('local_gb', 876) + self.host1.__setitem__('cpu_info', 1) self.host2 = Host() self.host2.__setitem__('name', 'host2') @@ -86,37 +90,43 @@ class SchedulerTestFunctions(unittest.TestCase): self.instance1 = Instance() for key, val in [ ('id', 1), ('host', 'host1'), ('hostname', 'i-12345'), ('state', power_state.RUNNING), ('project_id', 'testPJ'), - ('vcpus', 3), ('memory_mb', 1024), ('hdd_gb', 5) ]: + ('vcpus', 3), ('memory_mb', 1024), ('local_gb', 5) ]: self.instance1.__setitem__(key, val) self.instance2 = Instance() for key, val in [ ('id', 2), ('host', 'host1'), ('hostname', 'i-12345'), ('state', power_state.RUNNING), ('project_id', 'testPJ'), - ('vcpus', 3), ('memory_mb', 1024), ('hdd_gb', 5) ]: + ('vcpus', 3), ('memory_mb', 1024), ('local_gb', 5) ]: self.instance2.__setitem__(key, val) self.instance3 = Instance() for key, val in [ ('id', 3), ('host', 'host1'), ('hostname', 'i-12345'), ('state', power_state.RUNNING), ('project_id', 'testPJ2'), - ('vcpus', 1), ('memory_mb', 1024), ('hdd_gb', 5) ]: + ('vcpus', 1), ('memory_mb', 1024), ('local_gb', 5), + ('internal_id', 123456), ('state', 1), + ('state_description', 'running') ]: self.instance3.__setitem__(key, val) self.instance4 = Instance() for key, val in [ ('id', 4), ('host', 'host2'), ('hostname', 'i-12345'), ('state', power_state.RUNNING), ('project_id', 'testPJ2'), - ('vcpus', 1), ('memory_mb', 1024), ('local_gb', 5) ]: + ('vcpus', 1), ('memory_mb', 1024), ('local_gb', 5), + ('internal_id', 123456), ('state', 0), + ('state_description', 'running') ]: self.instance4.__setitem__(key, val) self.instance5 = Instance() for key, val in [ ('id', 5), ('host', 'host2'), ('hostname', 'i-12345'), ('state', power_state.RUNNING), ('project_id', 'testPJ2'), - ('vcpus', 1), ('memory_mb', 1024), ('local_gb', 5) ]: + ('vcpus', 1), ('memory_mb', 1024), ('local_gb', 5), + ('internal_id', 123456), ('state', 1), + ('state_description', 'migrating') ]: self.instance5.__setitem__(key, val) self.instance6 = Instance() - for key, val in [ ('id', 6), ('host', 'host1'), ('hostname', 'i-12345'), + for key, val in [ ('id', 6), ('host', 'host2'), ('hostname', 'i-12345'), ('state', power_state.RUNNING), ('project_id', 'testPJ2'), ('vcpus', 3), ('memory_mb', 1024), ('local_gb', 5) ]: self.instance6.__setitem__(key, val) @@ -129,7 +139,8 @@ class SchedulerTestFunctions(unittest.TestCase): self.instance8 = Instance() for key, val in [ ('id', 8), ('host', 'host1'), ('hostname', 'i-12345'), - ('state', power_state.RUNNING), ('project_id', 'testPJ2'), + ('state', power_state.RUNNING), + ('state_description', 'running'),('project_id', 'testPJ2'), ('vcpus', 1), ('memory_mb', 1024), ('local_gb', 866) ]: self.instance8.__setitem__(key, val) @@ -138,6 +149,10 @@ class SchedulerTestFunctions(unittest.TestCase): ('topic', 'compute')]: self.service1.__setitem__(key, val) + self.service2 = Service() + for key, val in [ ('id', 2), ('host', 'host2'), ('binary', 'nova-compute'), + ('topic', 'compute')]: + self.service1.__setitem__(key, val) def setMocks(self): self.ctxt = context.get_admin_context() @@ -147,10 +162,11 @@ class SchedulerTestFunctions(unittest.TestCase): db.instance_get_all_by_host = Mock(return_value = [self.instance4, self.instance5] ) # Mocks for live_migration - db.instance_get_by_internal_id = Mock(return_value = self.instance1) - # db.host_get_by_name <- defined above. db.service_get_all_by_topic = Mock(return_value = [self.service1] ) + self.manager.service_ip_up = Mock(return_value = True) rpc.call = Mock(return_value=1) + db.instance_set_state = Mock(return_value = True) + self.manager.driver.service_is_up = Mock(return_value = True) def check_format(self, val): """check result format of show_host_resource """ @@ -259,9 +275,12 @@ class SchedulerTestFunctions(unittest.TestCase): db.instance_get = Mock(return_value = self.instance6) try : - self.manager.has_enough_resource(self.ctxt, 'i-12345', 'host1') + self.manager.driver.has_enough_resource(self.ctxt, 'i-12345', 'host1') except exception.NotEmpty, e: - c1 = ( 0 < e.message.find('doesnt have enough resource') ) + # dont do e.message.find(), because the below message is occured. + # DeprecationWarning: BaseException.message has been deprecated + # as of Python 2.6 + c1 = ( 0 < str(e.args).find('doesnt have enough resource') ) self.assertTrue(c1, True) return False @@ -271,9 +290,9 @@ class SchedulerTestFunctions(unittest.TestCase): db.instance_get = Mock(return_value = self.instance7) try : - self.manager.has_enough_resource(self.ctxt, 'i-12345', 'host1') + self.manager.driver.has_enough_resource(self.ctxt, 'i-12345', 'host1') except exception.NotEmpty, e: - c1 = ( 0 <= e.message.find('doesnt have enough resource') ) + c1 = ( 0 <= str(e.args).find('doesnt have enough resource') ) self.assertTrue(c1, True) return False @@ -282,9 +301,9 @@ class SchedulerTestFunctions(unittest.TestCase): db.instance_get = Mock(return_value = self.instance8) try : - self.manager.has_enough_resource(self.ctxt, 'i-12345', 'host1') + self.manager.driver.has_enough_resource(self.ctxt, 'i-12345', 'host1') except exception.NotEmpty, e: - c1 = ( 0 <= e.message.find('doesnt have enough resource') ) + c1 = ( 0 <= str(e.args).find('doesnt have enough resource') ) self.assertTrue(c1, True) return False @@ -292,7 +311,7 @@ class SchedulerTestFunctions(unittest.TestCase): def test08(self): """08: everything goes well. (instance_get_all_by_host returns list)""" - ret= self.manager.has_enough_resource(self.ctxt, 'i-12345', 'host1') + ret= self.manager.driver.has_enough_resource(self.ctxt, 'i-12345', 'host1') self.assertEqual(ret, None) @@ -300,7 +319,7 @@ class SchedulerTestFunctions(unittest.TestCase): """09: everything goes well(instance_get_all_by_host returns[]). """ db.instance_get_all_by_host = Mock(return_value = [] ) - ret= self.manager.has_enough_resource(self.ctxt, 'i-12345', 'host1') + ret= self.manager.driver.has_enough_resource(self.ctxt, 'i-12345', 'host1') self.assertEqual(ret, None) @@ -308,91 +327,120 @@ class SchedulerTestFunctions(unittest.TestCase): def test10(self): - """10: instance_get_by_internal_id issue NotFound. """ + """10: instance_get issues NotFound. """ - # Mocks for has_enough_resource() - db.instance_get = Mock(return_value = self.instance8) - # Mocks for live_migration()db.instance_get_by_internal_id - # (any Mock is ok here. important mock is all above) - db.instance_get_by_internal_id = Mock(side_effect=exception.NotFound("ERR")) - + db.instance_get = Mock(side_effect=exception.NotFound("ERR")) self.assertRaises(exception.NotFound, - self.manager.live_migration, + self.manager.driver.schedule_live_migration, self.ctxt, 'i-12345', 'host1') - def test11(self): - """11: get NotFound exception when dest host not found on DB """ + """11: instance_get issues Unexpected error. """ - db.host_get_by_name = Mock( side_effect=exception.NotFound('ERR') ) - self.assertRaises(exception.NotFound, - self.manager.live_migration, + db.instance_get = Mock(side_effect=TypeError("ERR")) + self.assertRaises(TypeError, + self.manager.driver.schedule_live_migration, self.ctxt, 'i-12345', 'host1') - def test12(self): - """12: Destination host is not compute node """ - self.assertRaises(exception.Invalid, - self.manager.live_migration, - self.ctxt, - 'i-12345', - 'host2') + """12: instance state is not power_state.RUNNING. """ + + db.instance_get = Mock(return_value=self.instance4) + try : + self.manager.driver.schedule_live_migration(self.ctxt, 'i-12345', 'host1') + except exception.Invalid, e: + c1 = (0 <= str(e.args).find('is not running')) + self.assertTrue(c1, True) + return False + + def test13(self): + """13: instance state_description is not running. """ + + db.instance_get = Mock(return_value=self.instance5) + try : + self.manager.driver.schedule_live_migration(self.ctxt, 'i-12345', 'host1') + except exception.Invalid, e: + c1 = (0 <= str(e.args).find('is not running')) + self.assertTrue(c1, True) + return False + + def test14(self): + """14: dest is not compute node. + (dest is not included in the result of db.service_get_all_by_topic) + """ + try : + self.manager.driver.schedule_live_migration(self.ctxt, 'i-12345', 'host2') + except exception.Invalid, e: + c1 = (0 <= str(e.args).find('must be compute node')) + self.assertTrue(c1, True) + return False + def test15(self): + """ 15: dest is not alive.(service_is up returns False) """ + + self.manager.driver.service_is_up = Mock(return_value=False) + try : + self.manager.driver.schedule_live_migration(self.ctxt, 'i-12345', 'host2') + except exception.Invalid, e: + c1 = (0 <= str(e.args).find('is not alive')) + self.assertTrue(c1, True) + return False # Cannot test the case of hypervisor type difference and hypervisor # version difference, since we cannot set different mocks to same method.. - def test13(self): - """13: rpc.call raises RemoteError(Unexpected error occurs when executing compareCPU) """ + def test16(self): + """ 16: stored "cpuinfo" is not string """ + + try : + self.manager.driver.schedule_live_migration(self.ctxt, 'i-12345', 'host2') + except exception.Invalid, e: + c1 = (0 <= str(e.args).find('Unexpected err') ) + self.assertTrue(c1, True) + return False + + + def test17(self): + """17: rpc.call raises RemoteError(Unexpected error occurs when executing compareCPU) """ rpc.call = Mock(return_value = rpc.RemoteError(libvirt.libvirtError, 'val', 'traceback')) self.assertRaises(rpc.RemoteError, - self.manager.live_migration, + self.manager.driver.schedule_live_migration, self.ctxt, 'i-12345', - 'host1') + 'host2') - def test14(self): - """14: rpc.call returns 0 (cpu is not compatible between src and dest) """ + def test18(self): + """18: rpc.call returns 0 (cpu is not compatible between src and dest) """ rpc.call = Mock(return_value = 0) try : - self.manager.live_migration(self.ctxt, 'i-12345', 'host1') + self.manager.driver.schedule_live_migration(self.ctxt, 'i-12345', 'host2') except exception.Invalid, e: - c1 = ( 0 <= e.message.find('doesnt have compatibility to')) + c1 = ( 0 <= str(e.args).find('doesnt have compatibility to')) self.assertTrue(c1, True) return False - def test15(self): - """15: raise NotEmpty if host doesnt have enough resource. """ + def test19(self): + """19: raise NotEmpty if host doesnt have enough resource. """ - # Mocks for has_enough_resource() db.instance_get = Mock(return_value = self.instance8) - - # Mocks for live_migration() - db.instance_get_by_internal_id = Mock(return_value = self.instance8) - db.instance_set_state = Mock(return_value = True) - rpc_cast = Mock(return_value = True) - try : - self.manager.live_migration(self.ctxt, 'i-12345', 'host1') + self.manager.driver.schedule_live_migration(self.ctxt, 'i-12345', 'host2') except exception.NotEmpty, e: - c1 = ( 0 <= e.message.find('doesnt have enough resource') ) + c1 = ( 0 <= str(e.args).find('doesnt have enough resource') ) self.assertTrue(c1, True) return False - def test16(self): - """16: everything goes well. """ + def test20(self): + """20: everything goes well. """ - db.instance_get_by_internal_id = Mock(return_value = self.instance8) - db.instance_set_state = Mock(return_value = True) - rpc.cast = Mock(return_value = True) - - ret= self.manager.live_migration(self.ctxt, 'i-12345', 'host1') - self.assertEqual(ret, None) + #db.instance_get = Mock(return_value = self.instance8) + ret= self.manager.driver.schedule_live_migration(self.ctxt, 'i-12345', 'host2') + self.assertEqual(ret, self.instance8['host']) def tearDown(self): diff --git a/nova/livemigration_test/UT/testCase_UT.xls b/nova/livemigration_test/UT/testCase_UT.xls Binary files differindex f73e8c5aa..2850e70f0 100644 --- a/nova/livemigration_test/UT/testCase_UT.xls +++ b/nova/livemigration_test/UT/testCase_UT.xls diff --git a/nova/manager.py b/nova/manager.py index 5b61f7a4c..3d38504bd 100644 --- a/nova/manager.py +++ b/nova/manager.py @@ -55,7 +55,6 @@ from nova import utils from nova import flags from nova.db import base -from twisted.internet import defer FLAGS = flags.FLAGS @@ -67,10 +66,9 @@ class Manager(base.Base): self.host = host super(Manager, self).__init__(db_driver) - @defer.inlineCallbacks def periodic_tasks(self, context=None): """Tasks to be run at a periodic interval""" - yield + pass def init_host(self): """Do any initialization that needs to be run if this is a standalone diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 0fefd9415..931a89554 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -19,7 +19,6 @@ Implements vlans, bridges, and iptables rules using linux utilities. import logging import os -import signal # TODO(ja): does the definition of network_path belong here? @@ -46,41 +45,90 @@ flags.DEFINE_string('vlan_interface', 'eth0', 'network device for vlans') flags.DEFINE_string('dhcpbridge', _bin_file('nova-dhcpbridge'), 'location of nova-dhcpbridge') -flags.DEFINE_string('cc_host', utils.get_my_ip(), 'ip of api server') -flags.DEFINE_integer('cc_port', 8773, 'cloud controller port') -flags.DEFINE_string('routing_source_ip', '127.0.0.1', +flags.DEFINE_string('routing_source_ip', utils.get_my_ip(), 'Public IP of network host') flags.DEFINE_bool('use_nova_chains', False, 'use the nova_ routing chains instead of default') - -DEFAULT_PORTS = [("tcp", 80), ("tcp", 22), ("udp", 1194), ("tcp", 443)] +flags.DEFINE_string('dns_server', None, + 'if set, uses specific dns server for dnsmasq') +flags.DEFINE_string('dmz_cidr', '10.128.0.0/24', + 'dmz range that should be accepted') def metadata_forward(): """Create forwarding rule for metadata""" _confirm_rule("PREROUTING", "-t nat -s 0.0.0.0/0 " "-d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT " - "--to-destination %s:%s" % (FLAGS.cc_host, FLAGS.cc_port)) + "--to-destination %s:%s" % (FLAGS.cc_dmz, FLAGS.cc_port)) def init_host(): """Basic networking setup goes here""" + + if FLAGS.use_nova_chains: + _execute("sudo iptables -N nova_input", check_exit_code=False) + _execute("sudo iptables -D %s -j nova_input" % FLAGS.input_chain, + check_exit_code=False) + _execute("sudo iptables -A %s -j nova_input" % FLAGS.input_chain) + + _execute("sudo iptables -N nova_forward", check_exit_code=False) + _execute("sudo iptables -D FORWARD -j nova_forward", + check_exit_code=False) + _execute("sudo iptables -A FORWARD -j nova_forward") + + _execute("sudo iptables -N nova_output", check_exit_code=False) + _execute("sudo iptables -D OUTPUT -j nova_output", + check_exit_code=False) + _execute("sudo iptables -A OUTPUT -j nova_output") + + _execute("sudo iptables -t nat -N nova_prerouting", + check_exit_code=False) + _execute("sudo iptables -t nat -D PREROUTING -j nova_prerouting", + check_exit_code=False) + _execute("sudo iptables -t nat -A PREROUTING -j nova_prerouting") + + _execute("sudo iptables -t nat -N nova_postrouting", + check_exit_code=False) + _execute("sudo iptables -t nat -D POSTROUTING -j nova_postrouting", + check_exit_code=False) + _execute("sudo iptables -t nat -A POSTROUTING -j nova_postrouting") + + _execute("sudo iptables -t nat -N nova_snatting", + check_exit_code=False) + _execute("sudo iptables -t nat -D POSTROUTING -j nova_snatting", + check_exit_code=False) + _execute("sudo iptables -t nat -A POSTROUTING -j nova_snatting") + + _execute("sudo iptables -t nat -N nova_output", check_exit_code=False) + _execute("sudo iptables -t nat -D OUTPUT -j nova_output", + check_exit_code=False) + _execute("sudo iptables -t nat -A OUTPUT -j nova_output") + else: + # NOTE(vish): This makes it easy to ensure snatting rules always + # come after the accept rules in the postrouting chain + _execute("sudo iptables -t nat -N SNATTING", + check_exit_code=False) + _execute("sudo iptables -t nat -D POSTROUTING -j SNATTING", + check_exit_code=False) + _execute("sudo iptables -t nat -A POSTROUTING -j SNATTING") + # NOTE(devcamcar): Cloud public SNAT entries and the default # SNAT rule for outbound traffic. - _confirm_rule("POSTROUTING", "-t nat -s %s " + _confirm_rule("SNATTING", "-t nat -s %s " "-j SNAT --to-source %s" - % (FLAGS.fixed_range, FLAGS.routing_source_ip)) + % (FLAGS.fixed_range, FLAGS.routing_source_ip), append=True) - _confirm_rule("POSTROUTING", "-t nat -s %s -j MASQUERADE" % - FLAGS.fixed_range) + _confirm_rule("POSTROUTING", "-t nat -s %s -d %s -j ACCEPT" % + (FLAGS.fixed_range, FLAGS.dmz_cidr)) _confirm_rule("POSTROUTING", "-t nat -s %(range)s -d %(range)s -j ACCEPT" % {'range': FLAGS.fixed_range}) -def bind_floating_ip(floating_ip): +def bind_floating_ip(floating_ip, check_exit_code=True): """Bind ip to public interface""" _execute("sudo ip addr add %s dev %s" % (floating_ip, - FLAGS.public_interface)) + FLAGS.public_interface), + check_exit_code=check_exit_code) def unbind_floating_ip(floating_ip): @@ -102,27 +150,16 @@ def ensure_floating_forward(floating_ip, fixed_ip): """Ensure floating ip forwarding rule""" _confirm_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" % (floating_ip, fixed_ip)) - _confirm_rule("POSTROUTING", "-t nat -s %s -j SNAT --to %s" + _confirm_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" % (fixed_ip, floating_ip)) - # TODO(joshua): Get these from the secgroup datastore entries - _confirm_rule("FORWARD", "-d %s -p icmp -j ACCEPT" - % (fixed_ip)) - for (protocol, port) in DEFAULT_PORTS: - _confirm_rule("FORWARD", "-d %s -p %s --dport %s -j ACCEPT" - % (fixed_ip, protocol, port)) def remove_floating_forward(floating_ip, fixed_ip): """Remove forwarding for floating ip""" _remove_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" % (floating_ip, fixed_ip)) - _remove_rule("POSTROUTING", "-t nat -s %s -j SNAT --to %s" + _remove_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" % (fixed_ip, floating_ip)) - _remove_rule("FORWARD", "-d %s -p icmp -j ACCEPT" - % (fixed_ip)) - for (protocol, port) in DEFAULT_PORTS: - _remove_rule("FORWARD", "-d %s -p %s --dport %s -j ACCEPT" - % (fixed_ip, protocol, port)) def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): @@ -135,7 +172,7 @@ def ensure_vlan(vlan_num): """Create a vlan unless it already exists""" interface = "vlan%s" % vlan_num if not _device_exists(interface): - logging.debug("Starting VLAN inteface %s", interface) + logging.debug(_("Starting VLAN inteface %s"), interface) _execute("sudo vconfig set_name_type VLAN_PLUS_VID_NO_PAD") _execute("sudo vconfig add %s %s" % (FLAGS.vlan_interface, vlan_num)) _execute("sudo ifconfig %s up" % interface) @@ -145,7 +182,7 @@ def ensure_vlan(vlan_num): def ensure_bridge(bridge, interface, net_attrs=None): """Create a bridge unless it already exists""" if not _device_exists(bridge): - logging.debug("Starting Bridge interface for %s", interface) + logging.debug(_("Starting Bridge interface for %s"), interface) _execute("sudo brctl addbr %s" % bridge) _execute("sudo brctl setfd %s 0" % bridge) # _execute("sudo brctl setageing %s 10" % bridge) @@ -160,6 +197,15 @@ def ensure_bridge(bridge, interface, net_attrs=None): net_attrs['netmask'])) else: _execute("sudo ifconfig %s up" % bridge) + if FLAGS.use_nova_chains: + (out, err) = _execute("sudo iptables -N nova_forward", + check_exit_code=False) + if err != 'iptables: Chain already exists.\n': + # NOTE(vish): chain didn't exist link chain + _execute("sudo iptables -D FORWARD -j nova_forward", + check_exit_code=False) + _execute("sudo iptables -A FORWARD -j nova_forward") + _confirm_rule("FORWARD", "--in-interface %s -j ACCEPT" % bridge) _confirm_rule("FORWARD", "--out-interface %s -j ACCEPT" % bridge) @@ -202,9 +248,9 @@ def update_dhcp(context, network_id): _execute('sudo kill -HUP %d' % pid) return except Exception as exc: # pylint: disable-msg=W0703 - logging.debug("Hupping dnsmasq threw %s", exc) + logging.debug(_("Hupping dnsmasq threw %s"), exc) else: - logging.debug("Pid %d is stale, relaunching dnsmasq", pid) + logging.debug(_("Pid %d is stale, relaunching dnsmasq"), pid) # FLAGFILE and DNSMASQ_INTERFACE in env env = {'FLAGFILE': FLAGS.dhcpbridge_flagfile, @@ -236,13 +282,17 @@ def _device_exists(device): return not err -def _confirm_rule(chain, cmd): +def _confirm_rule(chain, cmd, append=False): """Delete and re-add iptables rule""" if FLAGS.use_nova_chains: chain = "nova_%s" % chain.lower() + if append: + loc = "-A" + else: + loc = "-I" _execute("sudo iptables --delete %s %s" % (chain, cmd), check_exit_code=False) - _execute("sudo iptables -I %s %s" % (chain, cmd)) + _execute("sudo iptables %s %s %s" % (loc, chain, cmd)) def _remove_rule(chain, cmd): @@ -265,6 +315,8 @@ def _dnsmasq_cmd(net): ' --dhcp-hostsfile=%s' % _dhcp_file(net['bridge'], 'conf'), ' --dhcp-script=%s' % FLAGS.dhcpbridge, ' --leasefile-ro'] + if FLAGS.dns_server: + cmd.append(' -h -R --server=%s' % FLAGS.dns_server) return ''.join(cmd) @@ -276,7 +328,7 @@ def _stop_dnsmasq(network): try: _execute('sudo kill -TERM %d' % pid) except Exception as exc: # pylint: disable-msg=W0703 - logging.debug("Killing dnsmasq threw %s", exc) + logging.debug(_("Killing dnsmasq threw %s"), exc) def _dhcp_file(bridge, kind): diff --git a/nova/network/manager.py b/nova/network/manager.py index a08b6094d..5ebc52e77 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -47,9 +47,9 @@ topologies. All of the network commands are issued to a subclass of import datetime import logging import math +import socket import IPy -from twisted.internet import defer from nova import context from nova import db @@ -57,6 +57,7 @@ from nova import exception from nova import flags from nova import manager from nova import utils +from nova import rpc FLAGS = flags.FLAGS @@ -88,6 +89,10 @@ flags.DEFINE_bool('update_dhcp_on_disassociate', False, 'Whether to update dhcp when fixed_ip is disassociated') flags.DEFINE_integer('fixed_ip_disassociate_timeout', 600, 'Seconds after which a deallocated ip is disassociated') +flags.DEFINE_string('network_host', socket.gethostname(), + 'Network host to use for ip allocation in flat modes') +flags.DEFINE_bool('fake_call', False, + 'If True, skip using the queue and make local calls') class AddressAlreadyAllocated(exception.Error): @@ -113,10 +118,20 @@ class NetworkManager(manager.Manager): ctxt = context.get_admin_context() for network in self.db.host_get_networks(ctxt, self.host): self._on_set_network_host(ctxt, network['id']) + floating_ips = self.db.floating_ip_get_all_by_host(ctxt, + self.host) + for floating_ip in floating_ips: + if floating_ip.get('fixed_ip', None): + fixed_address = floating_ip['fixed_ip']['address'] + # NOTE(vish): The False here is because we ignore the case + # that the ip is already bound. + self.driver.bind_floating_ip(floating_ip['address'], False) + self.driver.ensure_floating_forward(floating_ip['address'], + fixed_address) def set_network_host(self, context, network_id): """Safely sets the host of the network.""" - logging.debug("setting network host") + logging.debug(_("setting network host")) host = self.db.network_set_host(context, network_id, self.host) @@ -175,10 +190,10 @@ class NetworkManager(manager.Manager): fixed_ip_ref = self.db.fixed_ip_get_by_address(context, address) instance_ref = fixed_ip_ref['instance'] if not instance_ref: - raise exception.Error("IP %s leased that isn't associated" % + raise exception.Error(_("IP %s leased that isn't associated") % address) if instance_ref['mac_address'] != mac: - raise exception.Error("IP %s leased to bad mac %s vs %s" % + raise exception.Error(_("IP %s leased to bad mac %s vs %s") % (address, instance_ref['mac_address'], mac)) now = datetime.datetime.utcnow() self.db.fixed_ip_update(context, @@ -186,7 +201,8 @@ class NetworkManager(manager.Manager): {'leased': True, 'updated_at': now}) if not fixed_ip_ref['allocated']: - logging.warn("IP %s leased that was already deallocated", address) + logging.warn(_("IP %s leased that was already deallocated"), + address) def release_fixed_ip(self, context, mac, address): """Called by dhcp-bridge when ip is released.""" @@ -194,13 +210,13 @@ class NetworkManager(manager.Manager): fixed_ip_ref = self.db.fixed_ip_get_by_address(context, address) instance_ref = fixed_ip_ref['instance'] if not instance_ref: - raise exception.Error("IP %s released that isn't associated" % + raise exception.Error(_("IP %s released that isn't associated") % address) if instance_ref['mac_address'] != mac: - raise exception.Error("IP %s released from bad mac %s vs %s" % + raise exception.Error(_("IP %s released from bad mac %s vs %s") % (address, instance_ref['mac_address'], mac)) if not fixed_ip_ref['leased']: - logging.warn("IP %s released that was not leased", address) + logging.warn(_("IP %s released that was not leased"), address) self.db.fixed_ip_update(context, fixed_ip_ref['address'], {'leased': False}) @@ -213,8 +229,8 @@ class NetworkManager(manager.Manager): network_ref = self.db.fixed_ip_get_network(context, address) self.driver.update_dhcp(context, network_ref['id']) - def get_network(self, context): - """Get the network for the current context.""" + def get_network_host(self, context): + """Get the network host for the current context.""" raise NotImplementedError() def create_networks(self, context, num_networks, network_size, @@ -302,10 +318,6 @@ class FlatManager(NetworkManager): """Network is created manually.""" pass - def setup_fixed_ip(self, context, address): - """Currently no setup.""" - pass - def create_networks(self, context, cidr, num_networks, network_size, *args, **kwargs): """Create networks based on parameters.""" @@ -326,14 +338,25 @@ class FlatManager(NetworkManager): if network_ref: self._create_fixed_ips(context, network_ref['id']) - def get_network(self, context): - """Get the network for the current context.""" - # NOTE(vish): To support mutilple network hosts, This could randomly - # select from multiple networks instead of just - # returning the one. It could also potentially be done - # in the scheduler. - return self.db.network_get_by_bridge(context, - FLAGS.flat_network_bridge) + def get_network_host(self, context): + """Get the network host for the current context.""" + network_ref = self.db.network_get_by_bridge(context, + FLAGS.flat_network_bridge) + # NOTE(vish): If the network has no host, use the network_host flag. + # This could eventually be a a db lookup of some sort, but + # a flag is easy to handle for now. + host = network_ref['host'] + if not host: + topic = self.db.queue_get_for(context, + FLAGS.network_topic, + FLAGS.network_host) + if FLAGS.fake_call: + return self.set_network_host(context, network_ref['id']) + host = rpc.call(context, + FLAGS.network_topic, + {"method": "set_network_host", + "args": {"network_id": network_ref['id']}}) + return host def _on_set_network_host(self, context, network_id): """Called when this host becomes the host for a network.""" @@ -363,13 +386,18 @@ class FlatDHCPManager(FlatManager): if network_ref is None: network_ref = db.network_get_by_instance(context, instance_id) self.driver.ensure_bridge(network_ref['bridge'], - FLAGS.flat_interface, - network_ref) + FLAGS.flat_interface) - def setup_fixed_ip(self, context, address): + def allocate_fixed_ip(self, context, instance_id, *args, **kwargs): """Setup dhcp for this network.""" + address = super(FlatDHCPManager, self).allocate_fixed_ip(context, + instance_id, + *args, + **kwargs) network_ref = db.fixed_ip_get_network(context, address) - self.driver.update_dhcp(context, network_ref['id']) + if not FLAGS.fake_network: + self.driver.update_dhcp(context, network_ref['id']) + return address def deallocate_fixed_ip(self, context, address, *args, **kwargs): """Returns a fixed ip to the pool.""" @@ -400,10 +428,9 @@ class VlanManager(NetworkManager): instances in its subnet. """ - @defer.inlineCallbacks def periodic_tasks(self, context=None): """Tasks to be run at a periodic interval.""" - yield super(VlanManager, self).periodic_tasks(context) + super(VlanManager, self).periodic_tasks(context) now = datetime.datetime.utcnow() timeout = FLAGS.fixed_ip_disassociate_timeout time = now - datetime.timedelta(seconds=timeout) @@ -411,7 +438,7 @@ class VlanManager(NetworkManager): self.host, time) if num: - logging.debug("Dissassociated %s stale fixed ip(s)", num) + logging.debug(_("Dissassociated %s stale fixed ip(s)"), num) def init_host(self): """Do any initialization that needs to be run if this is a @@ -439,23 +466,15 @@ class VlanManager(NetworkManager): network_ref['id'], instance_id) self.db.fixed_ip_update(context, address, {'allocated': True}) + if not FLAGS.fake_network: + self.driver.update_dhcp(context, network_ref['id']) return address def deallocate_fixed_ip(self, context, address, *args, **kwargs): """Returns a fixed ip to the pool.""" self.db.fixed_ip_update(context, address, {'allocated': False}) - def setup_fixed_ip(self, context, address): - """Sets forwarding rules and dhcp for fixed ip.""" - fixed_ip_ref = self.db.fixed_ip_get_by_address(context, address) - network_ref = self.db.fixed_ip_get_network(context, address) - if self.db.instance_is_vpn(context, fixed_ip_ref['instance_id']): - self.driver.ensure_vlan_forward(network_ref['vpn_public_address'], - network_ref['vpn_public_port'], - network_ref['vpn_private_address']) - self.driver.update_dhcp(context, network_ref['id']) - - + #def setup_compute_network(self, context, instance_id): def setup_compute_network(self, context, instance_id, network_ref=None): """Sets up matching network for compute hosts.""" if network_ref is None: @@ -463,11 +482,6 @@ class VlanManager(NetworkManager): self.driver.ensure_vlan_bridge(network_ref['vlan'], network_ref['bridge']) - def restart_nets(self): - """Ensure the network for each user is enabled.""" - # TODO(vish): Implement this - pass - def create_networks(self, context, cidr, num_networks, network_size, vlan_start, vpn_start): """Create networks based on parameters.""" @@ -494,21 +508,45 @@ class VlanManager(NetworkManager): if network_ref: self._create_fixed_ips(context, network_ref['id']) - def get_network(self, context): + def get_network_host(self, context): """Get the network for the current context.""" - return self.db.project_get_network(context.elevated(), - context.project_id) + network_ref = self.db.project_get_network(context.elevated(), + context.project_id) + # NOTE(vish): If the network has no host, do a call to get an + # available host. This should be changed to go through + # the scheduler at some point. + host = network_ref['host'] + if not host: + if FLAGS.fake_call: + return self.set_network_host(context, network_ref['id']) + host = rpc.call(context, + FLAGS.network_topic, + {"method": "set_network_host", + "args": {"network_id": network_ref['id']}}) + + return host def _on_set_network_host(self, context, network_id): """Called when this host becomes the host for a network.""" network_ref = self.db.network_get(context, network_id) - net = {} - net['vpn_public_address'] = FLAGS.vpn_ip - db.network_update(context, network_id, net) + if not network_ref['vpn_public_address']: + net = {} + address = FLAGS.vpn_ip + net['vpn_public_address'] = address + db.network_update(context, network_id, net) + else: + address = network_ref['vpn_public_address'] self.driver.ensure_vlan_bridge(network_ref['vlan'], network_ref['bridge'], network_ref) - self.driver.update_dhcp(context, network_id) + # NOTE(vish): only ensure this forward if the address hasn't been set + # manually. + if address == FLAGS.vpn_ip: + self.driver.ensure_vlan_forward(FLAGS.vpn_ip, + network_ref['vpn_public_port'], + network_ref['vpn_private_address']) + if not FLAGS.fake_network: + self.driver.update_dhcp(context, network_id) @property def _bottom_reserved_ips(self): diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index c8920b00c..52257f69f 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -102,7 +102,7 @@ def _render_parts(value, write_cb): _render_parts(subsubvalue, write_cb) write_cb('</' + utils.utf8(name) + '>') else: - raise Exception("Unknown S3 value type %r", value) + raise Exception(_("Unknown S3 value type %r"), value) def get_argument(request, key, default_value): @@ -134,7 +134,7 @@ def get_context(request): check_type='s3') return context.RequestContext(user, project) except exception.Error as ex: - logging.debug("Authentication Failure: %s", ex) + logging.debug(_("Authentication Failure: %s"), ex) raise exception.NotAuthorized() @@ -227,7 +227,7 @@ class BucketResource(ErrorHandlingResource): def render_PUT(self, request): "Creates the bucket resource""" - logging.debug("Creating bucket %s", self.name) + logging.debug(_("Creating bucket %s"), self.name) logging.debug("calling bucket.Bucket.create(%r, %r)", self.name, request.context) @@ -237,7 +237,7 @@ class BucketResource(ErrorHandlingResource): def render_DELETE(self, request): """Deletes the bucket resource""" - logging.debug("Deleting bucket %s", self.name) + logging.debug(_("Deleting bucket %s"), self.name) bucket_object = bucket.Bucket(self.name) if not bucket_object.is_authorized(request.context): @@ -261,7 +261,9 @@ class ObjectResource(ErrorHandlingResource): Raises NotAuthorized if user in request context is not authorized to delete the object. """ - logging.debug("Getting object: %s / %s", self.bucket.name, self.name) + logging.debug(_("Getting object: %s / %s"), + self.bucket.name, + self.name) if not self.bucket.is_authorized(request.context): raise exception.NotAuthorized() @@ -279,7 +281,9 @@ class ObjectResource(ErrorHandlingResource): Raises NotAuthorized if user in request context is not authorized to delete the object. """ - logging.debug("Putting object: %s / %s", self.bucket.name, self.name) + logging.debug(_("Putting object: %s / %s"), + self.bucket.name, + self.name) if not self.bucket.is_authorized(request.context): raise exception.NotAuthorized() @@ -298,7 +302,7 @@ class ObjectResource(ErrorHandlingResource): authorized to delete the object. """ - logging.debug("Deleting object: %s / %s", + logging.debug(_("Deleting object: %s / %s"), self.bucket.name, self.name) @@ -394,17 +398,17 @@ class ImagesResource(resource.Resource): image_id = get_argument(request, 'image_id', u'') image_object = image.Image(image_id) if not image_object.is_authorized(request.context): - logging.debug("not authorized for render_POST in images") + logging.debug(_("not authorized for render_POST in images")) raise exception.NotAuthorized() operation = get_argument(request, 'operation', u'') if operation: # operation implies publicity toggle - logging.debug("handling publicity toggle") + logging.debug(_("handling publicity toggle")) image_object.set_public(operation == 'add') else: # other attributes imply update - logging.debug("update user fields") + logging.debug(_("update user fields")) clean_args = {} for arg in request.args.keys(): clean_args[arg] = request.args[arg][0] diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index 7292dbab8..abc28182e 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -21,7 +21,6 @@ Take uploaded bucket contents and register them as disk images (AMIs). Requires decryption using keys in the manifest. """ -# TODO(jesse): Got these from Euca2ools, will need to revisit them import binascii import glob @@ -29,7 +28,6 @@ import json import os import shutil import tarfile -import tempfile from xml.etree import ElementTree from nova import exception @@ -199,12 +197,17 @@ class Image(object): except: ramdisk_id = None + try: + arch = manifest.find("machine_configuration/architecture").text + except: + arch = 'x86_64' + info = { 'imageId': image_id, 'imageLocation': image_location, 'imageOwnerId': context.project_id, 'isPublic': False, # FIXME: grab public from manifest - 'architecture': 'x86_64', # FIXME: grab architecture from manifest + 'architecture': arch, 'imageType': image_type} if kernel_id: @@ -264,6 +267,7 @@ class Image(object): if err: raise exception.Error("Failed to decrypt initialization " "vector: %s" % err) + _out, err = utils.execute( 'openssl enc -d -aes-128-cbc -in %s -K %s -iv %s -out %s' % (encrypted_filename, key, iv, decrypted_filename), diff --git a/nova/process.py b/nova/process.py deleted file mode 100644 index b33df048b..000000000 --- a/nova/process.py +++ /dev/null @@ -1,209 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# Copyright 2010 FathomDB Inc. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Process pool using twisted threading -""" - -import logging -import StringIO - -from twisted.internet import defer -from twisted.internet import error -from twisted.internet import protocol -from twisted.internet import reactor - -from nova import flags -from nova.exception import ProcessExecutionError - -FLAGS = flags.FLAGS -flags.DEFINE_integer('process_pool_size', 4, - 'Number of processes to use in the process pool') - - -# This is based on _BackRelay from twister.internal.utils, but modified to -# capture both stdout and stderr, without odd stderr handling, and also to -# handle stdin -class BackRelayWithInput(protocol.ProcessProtocol): - """ - Trivial protocol for communicating with a process and turning its output - into the result of a L{Deferred}. - - @ivar deferred: A L{Deferred} which will be called back with all of stdout - and all of stderr as well (as a tuple). C{terminate_on_stderr} is true - and any bytes are received over stderr, this will fire with an - L{_ProcessExecutionError} instance and the attribute will be set to - C{None}. - - @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are - received over stderr, this attribute will refer to a L{Deferred} which - will be called back when the process ends. This C{Deferred} is also - associated with the L{_ProcessExecutionError} which C{deferred} fires - with earlier in this case so that users can determine when the process - has actually ended, in addition to knowing when bytes have been - received via stderr. - """ - - def __init__(self, deferred, cmd, started_deferred=None, - terminate_on_stderr=False, check_exit_code=True, - process_input=None): - self.deferred = deferred - self.cmd = cmd - self.stdout = StringIO.StringIO() - self.stderr = StringIO.StringIO() - self.started_deferred = started_deferred - self.terminate_on_stderr = terminate_on_stderr - self.check_exit_code = check_exit_code - self.process_input = process_input - self.on_process_ended = None - - def _build_execution_error(self, exit_code=None): - return ProcessExecutionError(cmd=self.cmd, - exit_code=exit_code, - stdout=self.stdout.getvalue(), - stderr=self.stderr.getvalue()) - - def errReceived(self, text): - self.stderr.write(text) - if self.terminate_on_stderr and (self.deferred is not None): - self.on_process_ended = defer.Deferred() - self.deferred.errback(self._build_execution_error()) - self.deferred = None - self.transport.loseConnection() - - def outReceived(self, text): - self.stdout.write(text) - - def processEnded(self, reason): - if self.deferred is not None: - stdout, stderr = self.stdout.getvalue(), self.stderr.getvalue() - exit_code = reason.value.exitCode - if self.check_exit_code and exit_code != 0: - self.deferred.errback(self._build_execution_error(exit_code)) - else: - try: - if self.check_exit_code: - reason.trap(error.ProcessDone) - self.deferred.callback((stdout, stderr)) - except: - # NOTE(justinsb): This logic is a little suspicious to me. - # If the callback throws an exception, then errback will - # be called also. However, this is what the unit tests - # test for. - exec_error = self._build_execution_error(exit_code) - self.deferred.errback(exec_error) - elif self.on_process_ended is not None: - self.on_process_ended.errback(reason) - - def connectionMade(self): - if self.started_deferred: - self.started_deferred.callback(self) - if self.process_input: - self.transport.write(str(self.process_input)) - self.transport.closeStdin() - - -def get_process_output(executable, args=None, env=None, path=None, - process_reactor=None, check_exit_code=True, - process_input=None, started_deferred=None, - terminate_on_stderr=False): - if process_reactor is None: - process_reactor = reactor - args = args and args or () - env = env and env and {} - deferred = defer.Deferred() - cmd = executable - if args: - cmd = " ".join([cmd] + args) - logging.debug("Running cmd: %s", cmd) - process_handler = BackRelayWithInput( - deferred, - cmd, - started_deferred=started_deferred, - check_exit_code=check_exit_code, - process_input=process_input, - terminate_on_stderr=terminate_on_stderr) - # NOTE(vish): commands come in as unicode, but self.executes needs - # strings or process.spawn raises a deprecation warning - executable = str(executable) - if not args is None: - args = [str(x) for x in args] - process_reactor.spawnProcess(process_handler, executable, - (executable,) + tuple(args), env, path) - return deferred - - -class ProcessPool(object): - """ A simple process pool implementation using Twisted's Process bits. - - This is pretty basic right now, but hopefully the API will be the correct - one so that it can be optimized later. - """ - def __init__(self, size=None): - self.size = size and size or FLAGS.process_pool_size - self._pool = defer.DeferredSemaphore(self.size) - - def simple_execute(self, cmd, **kw): - """ Weak emulation of the old utils.execute() function. - - This only exists as a way to quickly move old execute methods to - this new style of code. - - NOTE(termie): This will break on args with spaces in them. - """ - parsed = cmd.split(' ') - executable, args = parsed[0], parsed[1:] - return self.execute(executable, args, **kw) - - def execute(self, *args, **kw): - deferred = self._pool.acquire() - - def _associate_process(proto): - deferred.process = proto.transport - return proto.transport - - started = defer.Deferred() - started.addCallback(_associate_process) - kw.setdefault('started_deferred', started) - - deferred.process = None - deferred.started = started - - deferred.addCallback(lambda _: get_process_output(*args, **kw)) - deferred.addBoth(self._release) - return deferred - - def _release(self, retval=None): - self._pool.release() - return retval - - -class SharedPool(object): - _instance = None - - def __init__(self): - if SharedPool._instance is None: - self.__class__._instance = ProcessPool() - - def __getattr__(self, key): - return getattr(self._instance, key) - - -def simple_execute(cmd, **kwargs): - return SharedPool().simple_execute(cmd, **kwargs) diff --git a/nova/rpc.py b/nova/rpc.py index 86a29574f..844088348 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -25,18 +25,18 @@ import json import logging import sys import time +import traceback import uuid from carrot import connection as carrot_connection from carrot import messaging from eventlet import greenthread -from twisted.internet import defer -from twisted.internet import task +from nova import context from nova import exception from nova import fakerabbit from nova import flags -from nova import context +from nova import utils FLAGS = flags.FLAGS @@ -91,15 +91,15 @@ class Consumer(messaging.Consumer): self.failed_connection = False break except: # Catching all because carrot sucks - logging.exception("AMQP server on %s:%d is unreachable." \ - " Trying again in %d seconds." % ( + logging.exception(_("AMQP server on %s:%d is unreachable." + " Trying again in %d seconds.") % ( FLAGS.rabbit_host, FLAGS.rabbit_port, FLAGS.rabbit_retry_interval)) self.failed_connection = True if self.failed_connection: - logging.exception("Unable to connect to AMQP server" \ - " after %d tries. Shutting down." % FLAGS.rabbit_max_retries) + logging.exception(_("Unable to connect to AMQP server" + " after %d tries. Shutting down.") % FLAGS.rabbit_max_retries) sys.exit(1) def fetch(self, no_ack=None, auto_ack=None, enable_callbacks=False): @@ -116,29 +116,21 @@ class Consumer(messaging.Consumer): self.declare() super(Consumer, self).fetch(no_ack, auto_ack, enable_callbacks) if self.failed_connection: - logging.error("Reconnected to queue") + logging.error(_("Reconnected to queue")) self.failed_connection = False # NOTE(vish): This is catching all errors because we really don't # exceptions to be logged 10 times a second if some # persistent failure occurs. except Exception: # pylint: disable-msg=W0703 if not self.failed_connection: - logging.exception("Failed to fetch message from queue") + logging.exception(_("Failed to fetch message from queue")) self.failed_connection = True def attach_to_eventlet(self): """Only needed for unit tests!""" - def fetch_repeatedly(): - while True: - self.fetch(enable_callbacks=True) - greenthread.sleep(0.1) - greenthread.spawn(fetch_repeatedly) - - def attach_to_twisted(self): - """Attach a callback to twisted that fires 10 times a second""" - loop = task.LoopingCall(self.fetch, enable_callbacks=True) - loop.start(interval=0.1) - return loop + timer = utils.LoopingCall(self.fetch, enable_callbacks=True) + timer.start(0.1) + return timer class Publisher(messaging.Publisher): @@ -161,7 +153,7 @@ class TopicConsumer(Consumer): class AdapterConsumer(TopicConsumer): """Calls methods on a proxy object based on method and args""" def __init__(self, connection=None, topic="broadcast", proxy=None): - LOG.debug('Initing the Adapter Consumer for %s' % (topic)) + LOG.debug(_('Initing the Adapter Consumer for %s') % (topic)) self.proxy = proxy super(AdapterConsumer, self).__init__(connection=connection, topic=topic) @@ -176,7 +168,7 @@ class AdapterConsumer(TopicConsumer): Example: {'method': 'echo', 'args': {'value': 42}} """ - LOG.debug('received %s' % (message_data)) + LOG.debug(_('received %s') % (message_data)) msg_id = message_data.pop('_msg_id', None) ctxt = _unpack_context(message_data) @@ -189,18 +181,20 @@ class AdapterConsumer(TopicConsumer): # messages stay in the queue indefinitely, so for now # we just log the message and send an error string # back to the caller - LOG.warn('no method for message: %s' % (message_data)) - msg_reply(msg_id, 'No method for message: %s' % message_data) + LOG.warn(_('no method for message: %s') % (message_data)) + msg_reply(msg_id, _('No method for message: %s') % message_data) return node_func = getattr(self.proxy, str(method)) node_args = dict((str(k), v) for k, v in args.iteritems()) # NOTE(vish): magic is fun! - # pylint: disable-msg=W0142 - d = defer.maybeDeferred(node_func, context=ctxt, **node_args) - if msg_id: - d.addCallback(lambda rval: msg_reply(msg_id, rval, None)) - d.addErrback(lambda e: msg_reply(msg_id, None, e)) + try: + rval = node_func(context=ctxt, **node_args) + if msg_id: + msg_reply(msg_id, rval, None) + except Exception as e: + if msg_id: + msg_reply(msg_id, None, sys.exc_info()) return @@ -242,14 +236,16 @@ class DirectPublisher(Publisher): def msg_reply(msg_id, reply=None, failure=None): """Sends a reply or an error on the channel signified by msg_id - failure should be a twisted failure object""" + failure should be a sys.exc_info() tuple. + + """ if failure: - message = failure.getErrorMessage() - traceback = failure.getTraceback() - logging.error("Returning exception %s to caller", message) - logging.error(traceback) - failure = (failure.type.__name__, str(failure.value), traceback) - conn = Connection.instance() + message = str(failure[1]) + tb = traceback.format_exception(*failure) + logging.error(_("Returning exception %s to caller"), message) + logging.error(tb) + failure = (failure[0].__name__, str(failure[1]), tb) + conn = Connection.instance(True) publisher = DirectPublisher(connection=conn, msg_id=msg_id) try: publisher.send({'result': reply, 'failure': failure}) @@ -287,7 +283,7 @@ def _unpack_context(msg): if key.startswith('_context_'): value = msg.pop(key) context_dict[key[9:]] = value - LOG.debug('unpacked context: %s', context_dict) + LOG.debug(_('unpacked context: %s'), context_dict) return context.RequestContext.from_dict(context_dict) @@ -306,14 +302,13 @@ def _pack_context(msg, context): def call(context, topic, msg): """Sends a message on a topic and wait for a response""" - LOG.debug("Making asynchronous call...") + LOG.debug(_("Making asynchronous call...")) msg_id = uuid.uuid4().hex msg.update({'_msg_id': msg_id}) - LOG.debug("MSG_ID is %s" % (msg_id)) + LOG.debug(_("MSG_ID is %s") % (msg_id)) _pack_context(msg, context) class WaitMessage(object): - def __call__(self, data, message): """Acks message and sets result.""" message.ack() @@ -337,41 +332,15 @@ def call(context, topic, msg): except StopIteration: pass consumer.close() + # NOTE(termie): this is a little bit of a change from the original + # non-eventlet code where returning a Failure + # instance from a deferred call is very similar to + # raising an exception + if isinstance(wait_msg.result, Exception): + raise wait_msg.result return wait_msg.result -def call_twisted(context, topic, msg): - """Sends a message on a topic and wait for a response""" - LOG.debug("Making asynchronous call...") - msg_id = uuid.uuid4().hex - msg.update({'_msg_id': msg_id}) - LOG.debug("MSG_ID is %s" % (msg_id)) - _pack_context(msg, context) - - conn = Connection.instance() - d = defer.Deferred() - consumer = DirectConsumer(connection=conn, msg_id=msg_id) - - def deferred_receive(data, message): - """Acks message and callbacks or errbacks""" - message.ack() - if data['failure']: - return d.errback(RemoteError(*data['failure'])) - else: - return d.callback(data['result']) - - consumer.register_callback(deferred_receive) - injected = consumer.attach_to_twisted() - - # clean up after the injected listened and return x - d.addCallback(lambda x: injected.stop() and x or x) - - publisher = TopicPublisher(connection=conn, topic=topic) - publisher.send(msg) - publisher.close() - return d - - def cast(context, topic, msg): """Sends a message on a topic without waiting for a response""" LOG.debug("Making asynchronous cast...") @@ -384,7 +353,7 @@ def cast(context, topic, msg): def generic_response(message_data, message): """Logs a result and exits""" - LOG.debug('response %s', message_data) + LOG.debug(_('response %s'), message_data) message.ack() sys.exit(0) @@ -393,8 +362,8 @@ def send_message(topic, message, wait=True): """Sends a message for testing""" msg_id = uuid.uuid4().hex message.update({'_msg_id': msg_id}) - LOG.debug('topic is %s', topic) - LOG.debug('message %s', message) + LOG.debug(_('topic is %s'), topic) + LOG.debug(_('message %s'), message) if wait: consumer = messaging.Consumer(connection=Connection.instance(), diff --git a/nova/scheduler/chance.py b/nova/scheduler/chance.py index 7fd09b053..9deaa2777 100644 --- a/nova/scheduler/chance.py +++ b/nova/scheduler/chance.py @@ -34,5 +34,5 @@ class ChanceScheduler(driver.Scheduler): hosts = self.hosts_up(context, topic) if not hosts: - raise driver.NoValidHost("No hosts found") + raise driver.NoValidHost(_("No hosts found")) return hosts[int(random.random() * len(hosts))] diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index f271d573f..04061e38e 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -22,10 +22,14 @@ Scheduler base class that all Schedulers should inherit from """ import datetime +import logging from nova import db from nova import exception from nova import flags +from nova import rpc +from nova.api.ec2 import cloud +from nova.compute import power_state FLAGS = flags.FLAGS flags.DEFINE_integer('service_down_time', 60, @@ -58,4 +62,137 @@ class Scheduler(object): def schedule(self, context, topic, *_args, **_kwargs): """Must override at least this method for scheduler to work.""" - raise NotImplementedError("Must implement a fallback schedule") + raise NotImplementedError(_("Must implement a fallback schedule")) + + def schedule_live_migration(self, context, instance_id, dest): + """ live migration method """ + + # Whether instance exists and running + # try-catch clause is necessary because only internal_id is shown + # when NotFound exception occurs. it isnot understandable to admins. + try: + instance_ref = db.instance_get(context, instance_id) + ec2_id = instance_ref['hostname'] + internal_id = instance_ref['internal_id'] + except exception.NotFound, e: + msg = _('Unexpected error: instance is not found') + e.args += ('\n' + msg, ) + raise e + + # Checking instance state. + if power_state.RUNNING != instance_ref['state'] or \ + 'running' != instance_ref['state_description']: + msg = _('Instance(%s) is not running') + raise exception.Invalid(msg % ec2_id) + + # Checking destination host exists + dhost_ref = db.host_get_by_name(context, dest) + + # Checking whether The host where instance is running + # and dest is not same. + src = instance_ref['host'] + if dest == src: + msg = _('%s is where %s is running now. choose other host.') + raise exception.Invalid(msg % (dest, ec2_id)) + + # Checking dest is compute node. + services = db.service_get_all_by_topic(context, 'compute') + if dest not in [service.host for service in services]: + msg = _('%s must be compute node') + raise exception.Invalid(msg % dest) + + # Checking dest host is alive. + service = [service for service in services if service.host == dest] + service = service[0] + if not self.service_is_up(service): + msg = _('%s is not alive(time synchronize problem?)') + raise exception.Invalid(msg % dest) + + # NOTE(masumotok): Below pre-checkings are followed by + # http://wiki.libvirt.org/page/TodoPreMigrationChecks + + # Checking hypervisor is same. + orighost = instance_ref['launched_on'] + ohost_ref = db.host_get_by_name(context, orighost) + + otype = ohost_ref['hypervisor_type'] + dtype = dhost_ref['hypervisor_type'] + if otype != dtype: + msg = _('Different hypervisor type(%s->%s)') + raise exception.Invalid(msg % (otype, dtype)) + + # Checkng hypervisor version. + oversion = ohost_ref['hypervisor_version'] + dversion = dhost_ref['hypervisor_version'] + if oversion > dversion: + msg = _('Older hypervisor version(%s->%s)') + raise exception.Invalid(msg % (oversion, dversion)) + + # Checking cpuinfo. + cpuinfo = ohost_ref['cpu_info'] + if str != type(cpuinfo): + msg = _('Unexpected err: not found cpu_info for %s on DB.hosts') + raise exception.Invalid(msg % orighost) + + ret = rpc.call(context, + db.queue_get_for(context, FLAGS.compute_topic, dest), + {"method": 'compareCPU', + "args": {'xml': cpuinfo}}) + + if int != type(ret): + raise ret + + if 0 >= ret: + u = 'http://libvirt.org/html/libvirt-libvirt.html' + u += '#virCPUCompareResult' + msg = '%s doesnt have compatibility to %s(where %s launching at)\n' + msg += 'result:%d \n' + msg += 'Refer to %s' + msg = _(msg) + raise exception.Invalid(msg % (dest, src, ec2_id, ret, u)) + + # Checking dst host still has enough capacities. + self.has_enough_resource(context, instance_id, dest) + + # Changing instance_state. + db.instance_set_state(context, + instance_id, + power_state.PAUSED, + 'migrating') + + # Requesting live migration. + return src + + def has_enough_resource(self, context, instance_id, dest): + """ Check if destination host has enough resource for live migration""" + + # Getting instance information + instance_ref = db.instance_get(context, instance_id) + ec2_id = instance_ref['hostname'] + vcpus = instance_ref['vcpus'] + mem = instance_ref['memory_mb'] + hdd = instance_ref['local_gb'] + + # Gettin host information + host_ref = db.host_get_by_name(context, dest) + total_cpu = int(host_ref['vcpus']) + total_mem = int(host_ref['memory_mb']) + total_hdd = int(host_ref['local_gb']) + + instances_ref = db.instance_get_all_by_host(context, dest) + for i_ref in instances_ref: + total_cpu -= int(i_ref['vcpus']) + total_mem -= int(i_ref['memory_mb']) + total_hdd -= int(i_ref['local_gb']) + + # Checking host has enough information + logging.debug('host(%s) remains vcpu:%s mem:%s hdd:%s,' % + (dest, total_cpu, total_mem, total_hdd)) + logging.debug('instance(%s) has vcpu:%s mem:%s hdd:%s,' % + (ec2_id, vcpus, mem, hdd)) + + if total_cpu <= vcpus or total_mem <= mem or total_hdd <= hdd: + msg = '%s doesnt have enough resource for %s' % (dest, ec2_id) + raise exception.NotEmpty(msg) + + logging.debug(_('%s has enough resource for %s') % (dest, ec2_id)) diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index fe3ca9d5e..fd85241fc 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -30,8 +30,6 @@ from nova import manager from nova import rpc from nova import utils from nova import exception -from nova.api.ec2 import cloud -from nova.compute import power_state FLAGS = flags.FLAGS flags.DEFINE_string('scheduler_driver', @@ -68,119 +66,11 @@ class SchedulerManager(manager.Manager): db.queue_get_for(context, topic, host), {"method": method, "args": kwargs}) - logging.debug("Casting to %s %s for %s", topic, host, method) - - def live_migration(self, context, ec2_id, dest): - """ live migration method""" - - # (masumotok) below pre-checking is followed by - # http://wiki.libvirt.org/page/TodoPreMigrationChecks - - # 1. get instance id - internal_id = cloud.ec2_id_to_internal_id(ec2_id) - instance_ref = db.instance_get_by_internal_id(context, internal_id) - instance_id = instance_ref['id'] - - # 2. get src host and dst host - src = instance_ref['launched_on'] - shost_ref = db.host_get_by_name(context, src ) - dhost_ref = db.host_get_by_name(context, dest) - - # 3. dest should be compute - services = db.service_get_all_by_topic(context, 'compute') - logging.warn('%s' % [service.host for service in services]) - if dest not in [service.host for service in services] : - raise exception.Invalid('%s must be compute node' % dest) - - # 4. check hypervisor is same - shypervisor = shost_ref['hypervisor_type'] - dhypervisor = dhost_ref['hypervisor_type'] - if shypervisor != dhypervisor: - msg = 'Different hypervisor type(%s->%s)' % (shypervisor, dhypervisor) - raise exception.Invalid(msg) - - # 5. check hypervisor version - shypervisor = shost_ref['hypervisor_version'] - dhypervisor = dhost_ref['hypervisor_version'] - if shypervisor > dhypervisor: - msg = 'Older hypervisor version(%s->%s)' % (shypervisor, dhypervisor) - raise exception.Invalid(msg) - - # 6. check cpuinfo - cpuinfo = shost_ref['cpu_info'] - if str != type(cpuinfo): - msg = 'Unexpected err: no cpu_info for %s found on DB.hosts' % src - raise exception.Invalid(msg) - - logging.warn('cpuinfo %s %d' % (cpuinfo, len(cpuinfo))) - ret = rpc.call(context, - db.queue_get_for(context, FLAGS.compute_topic, dest), - {"method": 'compareCPU', - "args": {'xml': cpuinfo}}) - - if int != type(ret): - raise ret - - if 0 >= ret : - msg = '%s doesnt have compatibility to %s(where %s launching at)\n' \ - % (dest, src, ec2_id) - msg += 'result:%d \n' % ret - msg += 'Refer to %s' % \ - 'http://libvirt.org/html/libvirt-libvirt.html#virCPUCompareResult' - raise exception.Invalid(msg) - - # 7. check dst host still has enough capacities - self.has_enough_resource(context, instance_id, dest) - - # 8. change instance_state - db.instance_set_state(context, - instance_id, - power_state.PAUSED, - 'migrating') - - # 9. request live migration - host = instance_ref['host'] - rpc.cast(context, - db.queue_get_for(context, FLAGS.compute_topic, host), - {"method": 'live_migration', - "args": {'instance_id': instance_id, - 'dest': dest}}) - - - def has_enough_resource(self, context, instance_id, dest): - """ check if destination host has enough resource for live migration""" - - # get instance information - instance_ref = db.instance_get(context, instance_id) - ec2_id = instance_ref['hostname'] - vcpus = instance_ref['vcpus'] - mem = instance_ref['memory_mb'] - hdd = instance_ref['local_gb'] - - # get host information - host_ref = db.host_get_by_name(context, dest) - total_cpu = int(host_ref['vcpus']) - total_mem = int(host_ref['memory_mb']) - total_hdd = int(host_ref['local_gb']) - - instances_ref = db.instance_get_all_by_host(context, dest) - for i_ref in instances_ref: - total_cpu -= int(i_ref['vcpus']) - total_mem -= int(i_ref['memory_mb']) - total_hdd -= int(i_ref['local_gb']) - - # check host has enough information - logging.debug('host(%s) remains vcpu:%s mem:%s hdd:%s,' % - (dest, total_cpu, total_mem, total_hdd)) - logging.debug('instance(%s) has vcpu:%s mem:%s hdd:%s,' % - (ec2_id, total_cpu, total_mem, total_hdd)) - - if total_cpu <= vcpus or total_mem <= mem or total_hdd <= hdd: - msg = '%s doesnt have enough resource for %s' % (dest, ec2_id) - raise exception.NotEmpty(msg) - - logging.debug('%s has enough resource for %s' % (dest, ec2_id)) + logging.debug(_("Casting to %s %s for %s"), topic, host, method) + # NOTE (masumotok) : This method should be moved to nova.api.ec2.admin. + # Based on bear design summit discussion, + # just put this here for bexar release. def show_host_resource(self, context, host, *args): """ show the physical/usage resource given by hosts.""" @@ -191,12 +81,12 @@ class SchedulerManager(manager.Manager): except: raise - # get physical resource information + # Getting physical resource information h_resource = {'vcpus': host_ref['vcpus'], 'memory_mb': host_ref['memory_mb'], 'local_gb': host_ref['local_gb']} - # get usage resource information + # Getting usage resource information u_resource = {} instances_ref = db.instance_get_all_by_host(context, host_ref['name']) @@ -215,8 +105,8 @@ class SchedulerManager(manager.Manager): hdd = db.instance_get_disk_sum_by_host_and_project(context, host, p_id) - u_resource[p_id] = {'vcpus': vcpus, - 'memory_mb': mem, + u_resource[p_id] = {'vcpus': vcpus, + 'memory_mb': mem, 'local_gb': hdd} return {'ret': True, 'phy_resource': h_resource, 'usage': u_resource} diff --git a/nova/scheduler/simple.py b/nova/scheduler/simple.py index 7f5093656..f9171ab35 100644 --- a/nova/scheduler/simple.py +++ b/nova/scheduler/simple.py @@ -47,7 +47,7 @@ class SimpleScheduler(chance.ChanceScheduler): for result in results: (service, instance_cores) = result if instance_cores + instance_ref['vcpus'] > FLAGS.max_cores: - raise driver.NoValidHost("All hosts have too many cores") + raise driver.NoValidHost(_("All hosts have too many cores")) if self.service_is_up(service): # NOTE(vish): this probably belongs in the manager, if we # can generalize this somehow @@ -57,7 +57,7 @@ class SimpleScheduler(chance.ChanceScheduler): {'host': service['host'], 'scheduled_at': now}) return service['host'] - raise driver.NoValidHost("No hosts found") + raise driver.NoValidHost(_("No hosts found")) def schedule_create_volume(self, context, volume_id, *_args, **_kwargs): """Picks a host that is up and has the fewest volumes.""" @@ -66,7 +66,8 @@ class SimpleScheduler(chance.ChanceScheduler): for result in results: (service, volume_gigabytes) = result if volume_gigabytes + volume_ref['size'] > FLAGS.max_gigabytes: - raise driver.NoValidHost("All hosts have too many gigabytes") + raise driver.NoValidHost(_("All hosts have too many " + "gigabytes")) if self.service_is_up(service): # NOTE(vish): this probably belongs in the manager, if we # can generalize this somehow @@ -76,7 +77,7 @@ class SimpleScheduler(chance.ChanceScheduler): {'host': service['host'], 'scheduled_at': now}) return service['host'] - raise driver.NoValidHost("No hosts found") + raise driver.NoValidHost(_("No hosts found")) def schedule_set_network_host(self, context, *_args, **_kwargs): """Picks a host that is up and has the fewest networks.""" @@ -85,7 +86,7 @@ class SimpleScheduler(chance.ChanceScheduler): for result in results: (service, instance_count) = result if instance_count >= FLAGS.max_networks: - raise driver.NoValidHost("All hosts have too many networks") + raise driver.NoValidHost(_("All hosts have too many networks")) if self.service_is_up(service): return service['host'] - raise driver.NoValidHost("No hosts found") + raise driver.NoValidHost(_("No hosts found")) diff --git a/nova/server.py b/nova/server.py deleted file mode 100644 index a0ee54681..000000000 --- a/nova/server.py +++ /dev/null @@ -1,151 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Base functionality for nova daemons - gradually being replaced with twistd.py. -""" - -import daemon -from daemon import pidlockfile -import logging -import logging.handlers -import os -import signal -import sys -import time - -from nova import flags - - -FLAGS = flags.FLAGS -flags.DEFINE_bool('daemonize', False, 'daemonize this process') -# NOTE(termie): right now I am defaulting to using syslog when we daemonize -# it may be better to do something else -shrug- -# NOTE(Devin): I think we should let each process have its own log file -# and put it in /var/logs/nova/(appname).log -# This makes debugging much easier and cuts down on sys log -# clutter. -flags.DEFINE_bool('use_syslog', True, 'output to syslog when daemonizing') -flags.DEFINE_string('logfile', None, 'log file to output to') -flags.DEFINE_string('logdir', None, 'directory to keep log files in ' - '(will be prepended to $logfile)') -flags.DEFINE_string('pidfile', None, 'pid file to output to') -flags.DEFINE_string('working_directory', './', 'working directory...') -flags.DEFINE_integer('uid', os.getuid(), 'uid under which to run') -flags.DEFINE_integer('gid', os.getgid(), 'gid under which to run') - - -def stop(pidfile): - """ - Stop the daemon - """ - # Get the pid from the pidfile - try: - pid = int(open(pidfile, 'r').read().strip()) - except IOError: - message = "pidfile %s does not exist. Daemon not running?\n" - sys.stderr.write(message % pidfile) - return - - # Try killing the daemon process - try: - while 1: - os.kill(pid, signal.SIGTERM) - time.sleep(0.1) - except OSError, err: - err = str(err) - if err.find("No such process") > 0: - if os.path.exists(pidfile): - os.remove(pidfile) - else: - print str(err) - sys.exit(1) - - -def serve(name, main): - """Controller for server""" - argv = FLAGS(sys.argv) - - if not FLAGS.pidfile: - FLAGS.pidfile = '%s.pid' % name - - logging.debug("Full set of FLAGS: \n\n\n") - for flag in FLAGS: - logging.debug("%s : %s", flag, FLAGS.get(flag, None)) - - action = 'start' - if len(argv) > 1: - action = argv.pop() - - if action == 'stop': - stop(FLAGS.pidfile) - sys.exit() - elif action == 'restart': - stop(FLAGS.pidfile) - elif action == 'start': - pass - else: - print 'usage: %s [options] [start|stop|restart]' % argv[0] - sys.exit(1) - daemonize(argv, name, main) - - -def daemonize(args, name, main): - """Does the work of daemonizing the process""" - logging.getLogger('amqplib').setLevel(logging.WARN) - files_to_keep = [] - if FLAGS.daemonize: - logger = logging.getLogger() - formatter = logging.Formatter( - name + '(%(name)s): %(levelname)s %(message)s') - if FLAGS.use_syslog and not FLAGS.logfile: - syslog = logging.handlers.SysLogHandler(address='/dev/log') - syslog.setFormatter(formatter) - logger.addHandler(syslog) - files_to_keep.append(syslog.socket) - else: - if not FLAGS.logfile: - FLAGS.logfile = '%s.log' % name - if FLAGS.logdir: - FLAGS.logfile = os.path.join(FLAGS.logdir, FLAGS.logfile) - logfile = logging.FileHandler(FLAGS.logfile) - logfile.setFormatter(formatter) - logger.addHandler(logfile) - files_to_keep.append(logfile.stream) - stdin, stdout, stderr = None, None, None - else: - stdin, stdout, stderr = sys.stdin, sys.stdout, sys.stderr - - if FLAGS.verbose: - logging.getLogger().setLevel(logging.DEBUG) - else: - logging.getLogger().setLevel(logging.WARNING) - - with daemon.DaemonContext( - detach_process=FLAGS.daemonize, - working_directory=FLAGS.working_directory, - pidfile=pidlockfile.TimeoutPIDLockFile(FLAGS.pidfile, - acquire_timeout=1, - threaded=False), - stdin=stdin, - stdout=stdout, - stderr=stderr, - uid=FLAGS.uid, - gid=FLAGS.gid, - files_preserve=files_to_keep): - main(args) diff --git a/nova/service.py b/nova/service.py index 416448faa..04a60877e 100644 --- a/nova/service.py +++ b/nova/service.py @@ -17,21 +17,17 @@ # under the License. """ -A service is a very thin wrapper around a Manager object. It exposes the -manager's public methods to other components of the system via rpc. It will -report state periodically to the database and is responsible for initiating -any periodic tasts that need to be executed on a given host. - -This module contains Service, a generic baseclass for all workers. +Generic Node baseclass for all workers that run on hosts """ import inspect import logging import os +import sys -from twisted.internet import defer -from twisted.internet import task -from twisted.application import service +from eventlet import event +from eventlet import greenthread +from eventlet import greenpool from nova import context from nova import db @@ -50,8 +46,16 @@ flags.DEFINE_integer('periodic_interval', 60, 'seconds between running periodic tasks', lower_bound=1) +flags.DEFINE_string('pidfile', None, + 'pidfile to use for this service') + + +flags.DEFINE_flag(flags.HelpFlag()) +flags.DEFINE_flag(flags.HelpshortFlag()) +flags.DEFINE_flag(flags.HelpXMLFlag()) + -class Service(object, service.Service): +class Service(object): """Base class for workers that run on hosts.""" def __init__(self, host, binary, topic, manager, report_interval=None, @@ -64,8 +68,9 @@ class Service(object, service.Service): self.periodic_interval = periodic_interval super(Service, self).__init__(*args, **kwargs) self.saved_args, self.saved_kwargs = args, kwargs + self.timers = [] - def startService(self): # pylint: disable-msg C0103 + def start(self): manager_class = utils.import_class(self.manager_class_name) self.manager = manager_class(host=self.host, *self.saved_args, **self.saved_kwargs) @@ -87,26 +92,29 @@ class Service(object, service.Service): except exception.NotFound: self._create_service_ref(ctxt) - conn = rpc.Connection.instance() + conn1 = rpc.Connection.instance(new=True) + conn2 = rpc.Connection.instance(new=True) if self.report_interval: consumer_all = rpc.AdapterConsumer( - connection=conn, + connection=conn1, topic=self.topic, proxy=self) consumer_node = rpc.AdapterConsumer( - connection=conn, + connection=conn2, topic='%s.%s' % (self.topic, self.host), proxy=self) - consumer_all.attach_to_twisted() - consumer_node.attach_to_twisted() + self.timers.append(consumer_all.attach_to_eventlet()) + self.timers.append(consumer_node.attach_to_eventlet()) - pulse = task.LoopingCall(self.report_state) + pulse = utils.LoopingCall(self.report_state) pulse.start(interval=self.report_interval, now=False) + self.timers.append(pulse) if self.periodic_interval: - pulse = task.LoopingCall(self.periodic_tasks) - pulse.start(interval=self.periodic_interval, now=False) + periodic = utils.LoopingCall(self.periodic_tasks) + periodic.start(interval=self.periodic_interval, now=False) + self.timers.append(periodic) def _create_service_ref(self, context): service_ref = db.service_create(context, @@ -170,29 +178,32 @@ class Service(object, service.Service): report_interval = FLAGS.report_interval if not periodic_interval: periodic_interval = FLAGS.periodic_interval - logging.warn("Starting %s node", topic) + logging.warn(_("Starting %s node"), topic) service_obj = cls(host, binary, topic, manager, report_interval, periodic_interval) - # This is the parent service that twistd will be looking for when it - # parses this file, return it so that we can get it into globals. - application = service.Application(binary) - service_obj.setServiceParent(application) - return application + return service_obj def kill(self): """Destroy the service object in the datastore""" + self.stop() try: db.service_destroy(context.get_admin_context(), self.service_id) except exception.NotFound: - logging.warn("Service killed that has no database entry") + logging.warn(_("Service killed that has no database entry")) + + def stop(self): + for x in self.timers: + try: + x.stop() + except Exception: + pass + self.timers = [] - @defer.inlineCallbacks def periodic_tasks(self): """Tasks to be run at a periodic interval""" - yield self.manager.periodic_tasks(context.get_admin_context()) + self.manager.periodic_tasks(context.get_admin_context()) - @defer.inlineCallbacks def report_state(self): """Update the state of this service in the datastore.""" ctxt = context.get_admin_context() @@ -200,8 +211,8 @@ class Service(object, service.Service): try: service_ref = db.service_get(ctxt, self.service_id) except exception.NotFound: - logging.debug("The service database object disappeared, " - "Recreating it.") + logging.debug(_("The service database object disappeared, " + "Recreating it.")) self._create_service_ref(ctxt) service_ref = db.service_get(ctxt, self.service_id) @@ -212,11 +223,39 @@ class Service(object, service.Service): # TODO(termie): make this pattern be more elegant. if getattr(self, "model_disconnected", False): self.model_disconnected = False - logging.error("Recovered model server connection!") + logging.error(_("Recovered model server connection!")) # TODO(vish): this should probably only catch connection errors except Exception: # pylint: disable-msg=W0702 if not getattr(self, "model_disconnected", False): self.model_disconnected = True - logging.exception("model server went away") - yield + logging.exception(_("model server went away")) + + +def serve(*services): + argv = FLAGS(sys.argv) + + if not services: + services = [Service.create()] + + name = '_'.join(x.binary for x in services) + logging.debug("Serving %s" % name) + + logging.getLogger('amqplib').setLevel(logging.WARN) + + if FLAGS.verbose: + logging.getLogger().setLevel(logging.DEBUG) + else: + logging.getLogger().setLevel(logging.WARNING) + + logging.debug(_("Full set of FLAGS:")) + for flag in FLAGS: + logging.debug("%s : %s" % (flag, FLAGS.get(flag, None))) + + for x in services: + x.start() + + +def wait(): + while True: + greenthread.sleep(5) diff --git a/nova/service.py.THIS b/nova/service.py.THIS new file mode 100644 index 000000000..416448faa --- /dev/null +++ b/nova/service.py.THIS @@ -0,0 +1,222 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +A service is a very thin wrapper around a Manager object. It exposes the +manager's public methods to other components of the system via rpc. It will +report state periodically to the database and is responsible for initiating +any periodic tasts that need to be executed on a given host. + +This module contains Service, a generic baseclass for all workers. +""" + +import inspect +import logging +import os + +from twisted.internet import defer +from twisted.internet import task +from twisted.application import service + +from nova import context +from nova import db +from nova import exception +from nova import flags +from nova import rpc +from nova import utils + + +FLAGS = flags.FLAGS +flags.DEFINE_integer('report_interval', 10, + 'seconds between nodes reporting state to datastore', + lower_bound=1) + +flags.DEFINE_integer('periodic_interval', 60, + 'seconds between running periodic tasks', + lower_bound=1) + + +class Service(object, service.Service): + """Base class for workers that run on hosts.""" + + def __init__(self, host, binary, topic, manager, report_interval=None, + periodic_interval=None, *args, **kwargs): + self.host = host + self.binary = binary + self.topic = topic + self.manager_class_name = manager + self.report_interval = report_interval + self.periodic_interval = periodic_interval + super(Service, self).__init__(*args, **kwargs) + self.saved_args, self.saved_kwargs = args, kwargs + + def startService(self): # pylint: disable-msg C0103 + manager_class = utils.import_class(self.manager_class_name) + self.manager = manager_class(host=self.host, *self.saved_args, + **self.saved_kwargs) + self.manager.init_host() + self.model_disconnected = False + ctxt = context.get_admin_context() + + try: + host_ref = db.host_get_by_name(ctxt, self.host) + except exception.NotFound: + host_ref = db.host_create(ctxt, {'name': self.host}) + host_ref = self._update_host_ref(ctxt, host_ref) + + try: + service_ref = db.service_get_by_args(ctxt, + self.host, + self.binary) + self.service_id = service_ref['id'] + except exception.NotFound: + self._create_service_ref(ctxt) + + conn = rpc.Connection.instance() + if self.report_interval: + consumer_all = rpc.AdapterConsumer( + connection=conn, + topic=self.topic, + proxy=self) + consumer_node = rpc.AdapterConsumer( + connection=conn, + topic='%s.%s' % (self.topic, self.host), + proxy=self) + + consumer_all.attach_to_twisted() + consumer_node.attach_to_twisted() + + pulse = task.LoopingCall(self.report_state) + pulse.start(interval=self.report_interval, now=False) + + if self.periodic_interval: + pulse = task.LoopingCall(self.periodic_tasks) + pulse.start(interval=self.periodic_interval, now=False) + + def _create_service_ref(self, context): + service_ref = db.service_create(context, + {'host': self.host, + 'binary': self.binary, + 'topic': self.topic, + 'report_count': 0}) + self.service_id = service_ref['id'] + + def _update_host_ref(self, context, host_ref): + + if 0 <= self.manager_class_name.find('ComputeManager'): + vcpu = self.manager.driver.get_vcpu_number() + memory_mb = self.manager.get_memory_mb() + local_gb = self.manager.get_local_gb() + hypervisor = self.manager.driver.get_hypervisor_type() + version = self.manager.driver.get_hypervisor_version() + cpu_xml = self.manager.driver.get_cpu_xml() + + db.host_update(context, + host_ref['id'], + {'vcpus': vcpu, + 'memory_mb': memory_mb, + 'local_gb': local_gb, + 'hypervisor_type': hypervisor, + 'hypervisor_version': version, + 'cpu_info':cpu_xml }) + return host_ref + + def __getattr__(self, key): + manager = self.__dict__.get('manager', None) + return getattr(manager, key) + + @classmethod + def create(cls, + host=None, + binary=None, + topic=None, + manager=None, + report_interval=None, + periodic_interval=None): + """Instantiates class and passes back application object. + + Args: + host, defaults to FLAGS.host + binary, defaults to basename of executable + topic, defaults to bin_name - "nova-" part + manager, defaults to FLAGS.<topic>_manager + report_interval, defaults to FLAGS.report_interval + periodic_interval, defaults to FLAGS.periodic_interval + """ + if not host: + host = FLAGS.host + if not binary: + binary = os.path.basename(inspect.stack()[-1][1]) + if not topic: + topic = binary.rpartition("nova-")[2] + if not manager: + manager = FLAGS.get('%s_manager' % topic, None) + if not report_interval: + report_interval = FLAGS.report_interval + if not periodic_interval: + periodic_interval = FLAGS.periodic_interval + logging.warn("Starting %s node", topic) + service_obj = cls(host, binary, topic, manager, + report_interval, periodic_interval) + + # This is the parent service that twistd will be looking for when it + # parses this file, return it so that we can get it into globals. + application = service.Application(binary) + service_obj.setServiceParent(application) + return application + + def kill(self): + """Destroy the service object in the datastore""" + try: + db.service_destroy(context.get_admin_context(), self.service_id) + except exception.NotFound: + logging.warn("Service killed that has no database entry") + + @defer.inlineCallbacks + def periodic_tasks(self): + """Tasks to be run at a periodic interval""" + yield self.manager.periodic_tasks(context.get_admin_context()) + + @defer.inlineCallbacks + def report_state(self): + """Update the state of this service in the datastore.""" + ctxt = context.get_admin_context() + try: + try: + service_ref = db.service_get(ctxt, self.service_id) + except exception.NotFound: + logging.debug("The service database object disappeared, " + "Recreating it.") + self._create_service_ref(ctxt) + service_ref = db.service_get(ctxt, self.service_id) + + db.service_update(ctxt, + self.service_id, + {'report_count': service_ref['report_count'] + 1}) + + # TODO(termie): make this pattern be more elegant. + if getattr(self, "model_disconnected", False): + self.model_disconnected = False + logging.error("Recovered model server connection!") + + # TODO(vish): this should probably only catch connection errors + except Exception: # pylint: disable-msg=W0702 + if not getattr(self, "model_disconnected", False): + self.model_disconnected = True + logging.exception("model server went away") + yield diff --git a/nova/test.py b/nova/test.py index 5c2a72819..db5826c04 100644 --- a/nova/test.py +++ b/nova/test.py @@ -25,11 +25,12 @@ and some black magic for inline callbacks. import datetime import sys import time +import unittest import mox import stubout from twisted.internet import defer -from twisted.trial import unittest +from twisted.trial import unittest as trial_unittest from nova import context from nova import db @@ -37,9 +38,12 @@ from nova import fakerabbit from nova import flags from nova import rpc from nova.network import manager as network_manager +from nova.tests import fake_flags FLAGS = flags.FLAGS +flags.DEFINE_bool('flush_db', True, + 'Flush the database before running fake tests') flags.DEFINE_bool('fake_tests', True, 'should we use everything for testing') @@ -55,11 +59,11 @@ def skip_if_fake(func): return _skipper -class TrialTestCase(unittest.TestCase): +class TestCase(unittest.TestCase): """Test case base class for all unit tests""" def setUp(self): """Run before each test method to initialize test environment""" - super(TrialTestCase, self).setUp() + super(TestCase, self).setUp() # NOTE(vish): We need a better method for creating fixtures for tests # now that we have some required db setup for the system # to work properly. @@ -94,7 +98,87 @@ class TrialTestCase(unittest.TestCase): db.fixed_ip_disassociate_all_by_timeout(ctxt, FLAGS.host, self.start) db.network_disassociate_all(ctxt) - rpc.Consumer.attach_to_twisted = self.originalAttach + rpc.Consumer.attach_to_eventlet = self.originalAttach + for x in self.injected: + try: + x.stop() + except AssertionError: + pass + + if FLAGS.fake_rabbit: + fakerabbit.reset_all() + + db.security_group_destroy_all(ctxt) + super(TestCase, self).tearDown() + finally: + self.reset_flags() + + def flags(self, **kw): + """Override flag variables for a test""" + for k, v in kw.iteritems(): + if k in self.flag_overrides: + self.reset_flags() + raise Exception( + 'trying to override already overriden flag: %s' % k) + self.flag_overrides[k] = getattr(FLAGS, k) + setattr(FLAGS, k, v) + + def reset_flags(self): + """Resets all flag variables for the test. Runs after each test""" + FLAGS.Reset() + for k, v in self._original_flags.iteritems(): + setattr(FLAGS, k, v) + + def _monkey_patch_attach(self): + self.originalAttach = rpc.Consumer.attach_to_eventlet + + def _wrapped(innerSelf): + rv = self.originalAttach(innerSelf) + self.injected.append(rv) + return rv + + _wrapped.func_name = self.originalAttach.func_name + rpc.Consumer.attach_to_eventlet = _wrapped + + +class TrialTestCase(trial_unittest.TestCase): + """Test case base class for all unit tests""" + def setUp(self): + """Run before each test method to initialize test environment""" + super(TrialTestCase, self).setUp() + # NOTE(vish): We need a better method for creating fixtures for tests + # now that we have some required db setup for the system + # to work properly. + self.start = datetime.datetime.utcnow() + ctxt = context.get_admin_context() + if db.network_count(ctxt) != 5: + network_manager.VlanManager().create_networks(ctxt, + FLAGS.fixed_range, + 5, 16, + FLAGS.vlan_start, + FLAGS.vpn_start) + + # emulate some of the mox stuff, we can't use the metaclass + # because it screws with our generators + self.mox = mox.Mox() + self.stubs = stubout.StubOutForTesting() + self.flag_overrides = {} + self.injected = [] + self._original_flags = FLAGS.FlagValuesDict() + + def tearDown(self): + """Runs after each test method to finalize/tear down test + environment.""" + try: + self.mox.UnsetStubs() + self.stubs.UnsetAll() + self.stubs.SmartUnsetAll() + self.mox.VerifyAll() + # NOTE(vish): Clean up any ips associated during the test. + ctxt = context.get_admin_context() + db.fixed_ip_disassociate_all_by_timeout(ctxt, FLAGS.host, + self.start) + db.network_disassociate_all(ctxt) for x in self.injected: try: x.stop() @@ -147,14 +231,3 @@ class TrialTestCase(unittest.TestCase): return d _wrapped.func_name = func.func_name return _wrapped - - def _monkey_patch_attach(self): - self.originalAttach = rpc.Consumer.attach_to_twisted - - def _wrapped(innerSelf): - rv = self.originalAttach(innerSelf) - self.injected.append(rv) - return rv - - _wrapped.func_name = self.originalAttach.func_name - rpc.Consumer.attach_to_twisted = _wrapped diff --git a/nova/tests/__init__.py b/nova/tests/__init__.py index aaf213923..8dc87d0e2 100644 --- a/nova/tests/__init__.py +++ b/nova/tests/__init__.py @@ -29,3 +29,8 @@ .. moduleauthor:: Manish Singh <yosh@gimp.org> .. moduleauthor:: Andy Smith <andy@anarkystic.com> """ + +# See http://code.google.com/p/python-nose/issues/detail?id=373 +# The code below enables nosetests to work with i18n _() blocks +import __builtin__ +setattr(__builtin__, '_', lambda x: x) diff --git a/nova/tests/api/__init__.py b/nova/tests/api/__init__.py index 9caa8c9d0..e69de29bb 100644 --- a/nova/tests/api/__init__.py +++ b/nova/tests/api/__init__.py @@ -1,81 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Test for the root WSGI middleware for all API controllers. -""" - -import unittest - -import stubout -import webob -import webob.dec - -import nova.exception -from nova import api -from nova.tests.api.fakes import APIStub - - -class Test(unittest.TestCase): - - def setUp(self): - self.stubs = stubout.StubOutForTesting() - - def tearDown(self): - self.stubs.UnsetAll() - - def _request(self, url, subdomain, **kwargs): - environ_keys = {'HTTP_HOST': '%s.example.com' % subdomain} - environ_keys.update(kwargs) - req = webob.Request.blank(url, environ_keys) - return req.get_response(api.API('ec2')) - - def test_openstack(self): - self.stubs.Set(api.openstack, 'API', APIStub) - result = self._request('/v1.0/cloud', 'api') - self.assertEqual(result.body, "/cloud") - - def test_ec2(self): - self.stubs.Set(api.ec2, 'API', APIStub) - result = self._request('/services/cloud', 'ec2') - self.assertEqual(result.body, "/cloud") - - def test_not_found(self): - self.stubs.Set(api.ec2, 'API', APIStub) - self.stubs.Set(api.openstack, 'API', APIStub) - result = self._request('/test/cloud', 'ec2') - self.assertNotEqual(result.body, "/cloud") - - def test_query_api_versions(self): - result = self._request('/', 'api') - self.assertTrue('CURRENT' in result.body) - - def test_metadata(self): - def go(url): - result = self._request(url, 'ec2', REMOTE_ADDR='128.192.151.2') - # Each should get to the ORM layer and fail to find the IP - self.assertRaises(nova.exception.NotFound, go, '/latest/') - self.assertRaises(nova.exception.NotFound, go, '/2009-04-04/') - self.assertRaises(nova.exception.NotFound, go, '/1.0/') - - def test_ec2_root(self): - result = self._request('/', 'ec2') - self.assertTrue('2007-12-15\n' in result.body) - - -if __name__ == '__main__': - unittest.main() diff --git a/nova/tests/api/openstack/__init__.py b/nova/tests/api/openstack/__init__.py index 2e357febe..9e183bd0d 100644 --- a/nova/tests/api/openstack/__init__.py +++ b/nova/tests/api/openstack/__init__.py @@ -17,11 +17,16 @@ import unittest -from nova.api.openstack import limited -from nova.api.openstack import RateLimitingMiddleware +from nova import context +from nova import flags +from nova.api.openstack.ratelimiting import RateLimitingMiddleware +from nova.api.openstack.common import limited from nova.tests.api.fakes import APIStub +from nova import utils from webob import Request +FLAGS = flags.FLAGS + class RateLimitingMiddlewareTest(unittest.TestCase): @@ -46,6 +51,8 @@ class RateLimitingMiddlewareTest(unittest.TestCase): def exhaust(self, middleware, method, url, username, times): req = Request.blank(url, dict(REQUEST_METHOD=method), headers={'X-Auth-User': username}) + req.environ['nova.context'] = context.RequestContext(username, + username) for i in range(times): resp = req.get_response(middleware) self.assertEqual(resp.status_int, 200) @@ -62,7 +69,7 @@ class RateLimitingMiddlewareTest(unittest.TestCase): middleware = RateLimitingMiddleware(APIStub()) self.exhaust(middleware, 'POST', '/servers/4', 'usr1', 10) self.exhaust(middleware, 'POST', '/images/4', 'usr2', 10) - self.assertTrue(set(middleware.limiter._levels) == + self.assertTrue(set(middleware.limiter._levels) == \ set(['usr1:POST', 'usr1:POST servers', 'usr2:POST'])) def test_POST_servers_action_correctly_ratelimited(self): diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 7c0343942..79663e43a 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -24,12 +24,16 @@ import webob import webob.dec from nova import auth -from nova import utils -from nova import flags +from nova import context from nova import exception as exc +from nova import flags +from nova import utils import nova.api.openstack.auth -from nova.image import service +from nova.api.openstack import auth +from nova.api.openstack import ratelimiting from nova.image import glance +from nova.image import local +from nova.image import service from nova.tests import fake_flags from nova.wsgi import Router @@ -50,15 +54,16 @@ class FakeRouter(Router): return res -def fake_auth_init(self): +def fake_auth_init(self, application): self.db = FakeAuthDatabase() self.context = Context() self.auth = FakeAuthManager() + self.application = application @webob.dec.wsgify def fake_wsgi(self, req): - req.environ['nova.context'] = dict(user=dict(id=1)) + req.environ['nova.context'] = context.RequestContext(1, 1) if req.body: req.environ['inst_dict'] = json.loads(req.body) return self.application @@ -67,36 +72,35 @@ def fake_wsgi(self, req): def stub_out_key_pair_funcs(stubs): def key_pair(context, user_id): return [dict(name='key', public_key='public_key')] - stubs.Set(nova.db.api, 'key_pair_get_all_by_user', - key_pair) + stubs.Set(nova.db, 'key_pair_get_all_by_user', key_pair) def stub_out_image_service(stubs): def fake_image_show(meh, context, id): return dict(kernelId=1, ramdiskId=1) - stubs.Set(nova.image.local.LocalImageService, 'show', fake_image_show) + stubs.Set(local.LocalImageService, 'show', fake_image_show) def stub_out_auth(stubs): def fake_auth_init(self, app): self.application = app - stubs.Set(nova.api.openstack.AuthMiddleware, + stubs.Set(nova.api.openstack.auth.AuthMiddleware, '__init__', fake_auth_init) - stubs.Set(nova.api.openstack.AuthMiddleware, + stubs.Set(nova.api.openstack.auth.AuthMiddleware, '__call__', fake_wsgi) def stub_out_rate_limiting(stubs): def fake_rate_init(self, app): - super(nova.api.openstack.RateLimitingMiddleware, self).__init__(app) + super(ratelimiting.RateLimitingMiddleware, self).__init__(app) self.application = app - stubs.Set(nova.api.openstack.RateLimitingMiddleware, + stubs.Set(nova.api.openstack.ratelimiting.RateLimitingMiddleware, '__init__', fake_rate_init) - stubs.Set(nova.api.openstack.RateLimitingMiddleware, + stubs.Set(nova.api.openstack.ratelimiting.RateLimitingMiddleware, '__call__', fake_wsgi) @@ -172,6 +176,12 @@ class FakeToken(object): setattr(self, k, v) +class FakeRequestContext(object): + def __init__(self, user, project, *args, **kwargs): + self.user_id = 1 + self.project_id = 1 + + class FakeAuthDatabase(object): data = {} diff --git a/nova/tests/api/openstack/test_adminapi.py b/nova/tests/api/openstack/test_adminapi.py new file mode 100644 index 000000000..1b2e1654d --- /dev/null +++ b/nova/tests/api/openstack/test_adminapi.py @@ -0,0 +1,61 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + +import stubout +import webob + +import nova.api +from nova import flags +from nova.tests.api.openstack import fakes + +FLAGS = flags.FLAGS + + +class AdminAPITest(unittest.TestCase): + def setUp(self): + self.stubs = stubout.StubOutForTesting() + fakes.FakeAuthManager.auth_data = {} + fakes.FakeAuthDatabase.data = {} + fakes.stub_out_networking(self.stubs) + fakes.stub_out_rate_limiting(self.stubs) + fakes.stub_out_auth(self.stubs) + self.allow_admin = FLAGS.allow_admin_api + + def tearDown(self): + self.stubs.UnsetAll() + FLAGS.allow_admin_api = self.allow_admin + + def test_admin_enabled(self): + FLAGS.allow_admin_api = True + # We should still be able to access public operations. + req = webob.Request.blank('/v1.0/flavors') + res = req.get_response(nova.api.API('os')) + self.assertEqual(res.status_int, 200) + # TODO: Confirm admin operations are available. + + def test_admin_disabled(self): + FLAGS.allow_admin_api = False + # We should still be able to access public operations. + req = webob.Request.blank('/v1.0/flavors') + res = req.get_response(nova.api.API('os')) + self.assertEqual(res.status_int, 200) + # TODO: Confirm admin operations are unavailable. + +if __name__ == '__main__': + unittest.main() diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 14e720be4..489a1dfbf 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -26,6 +26,7 @@ import nova.api import nova.api.openstack.auth import nova.auth.manager from nova import auth +from nova import context from nova.tests.api.openstack import fakes @@ -33,8 +34,9 @@ class Test(unittest.TestCase): def setUp(self): self.stubs = stubout.StubOutForTesting() - self.stubs.Set(nova.api.openstack.auth.BasicApiAuthManager, + self.stubs.Set(nova.api.openstack.auth.AuthMiddleware, '__init__', fakes.fake_auth_init) + self.stubs.Set(context, 'RequestContext', fakes.FakeRequestContext) fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} fakes.stub_out_rate_limiting(self.stubs) @@ -129,8 +131,9 @@ class Test(unittest.TestCase): class TestLimiter(unittest.TestCase): def setUp(self): self.stubs = stubout.StubOutForTesting() - self.stubs.Set(nova.api.openstack.auth.BasicApiAuthManager, + self.stubs.Set(nova.api.openstack.auth.AuthMiddleware, '__init__', fakes.fake_auth_init) + self.stubs.Set(context, 'RequestContext', fakes.FakeRequestContext) fakes.FakeAuthManager.auth_data = {} fakes.FakeAuthDatabase.data = {} fakes.stub_out_networking(self.stubs) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 44ac8f342..3820f5f27 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -47,12 +47,25 @@ def return_security_group(context, instance_id, security_group_id): pass +def instance_update(context, instance_id, kwargs): + return stub_instance(instance_id) + + +def instance_address(context, instance_id): + return None + + def stub_instance(id, user_id=1): - return Instance(id=id + 123456, state=0, image_id=10, user_id=user_id, + return Instance(id=int(id) + 123456, state=0, image_id=10, user_id=user_id, display_name='server%s' % id, internal_id=id) +def fake_compute_api(cls, req, id): + return True + + class ServersTest(unittest.TestCase): + def setUp(self): self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.auth_data = {} @@ -69,9 +82,20 @@ class ServersTest(unittest.TestCase): return_servers) self.stubs.Set(nova.db.api, 'instance_add_security_group', return_security_group) + self.stubs.Set(nova.db.api, 'instance_update', instance_update) + self.stubs.Set(nova.db.api, 'instance_get_fixed_address', + instance_address) + self.stubs.Set(nova.db.api, 'instance_get_floating_address', + instance_address) + self.stubs.Set(nova.compute.api.ComputeAPI, 'pause', + fake_compute_api) + self.stubs.Set(nova.compute.api.ComputeAPI, 'unpause', + fake_compute_api) + self.allow_admin = FLAGS.allow_admin_api def tearDown(self): self.stubs.UnsetAll() + FLAGS.allow_admin_api = self.allow_admin def test_get_server_by_id(self): req = webob.Request.blank('/v1.0/servers/1') @@ -93,11 +117,11 @@ class ServersTest(unittest.TestCase): i += 1 def test_create_instance(self): - def server_update(context, id, params): - pass - def instance_create(context, inst): - return {'id': 1, 'internal_id': 1} + return {'id': 1, 'internal_id': 1, 'display_name': ''} + + def server_update(context, id, params): + return instance_create(context, id) def fake_method(*args, **kwargs): pass @@ -198,6 +222,30 @@ class ServersTest(unittest.TestCase): self.assertEqual(s['imageId'], 10) i += 1 + def test_server_pause(self): + FLAGS.allow_admin_api = True + body = dict(server=dict( + name='server_test', imageId=2, flavorId=2, metadata={}, + personality={})) + req = webob.Request.blank('/v1.0/servers/1/pause') + req.method = 'POST' + req.content_type = 'application/json' + req.body = json.dumps(body) + res = req.get_response(nova.api.API('os')) + self.assertEqual(res.status_int, 202) + + def test_server_unpause(self): + FLAGS.allow_admin_api = True + body = dict(server=dict( + name='server_test', imageId=2, flavorId=2, metadata={}, + personality={})) + req = webob.Request.blank('/v1.0/servers/1/unpause') + req.method = 'POST' + req.content_type = 'application/json' + req.body = json.dumps(body) + res = req.get_response(nova.api.API('os')) + self.assertEqual(res.status_int, 202) + def test_server_reboot(self): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, diff --git a/nova/tests/api/test.py b/nova/tests/api/test.py new file mode 100644 index 000000000..9caa8c9d0 --- /dev/null +++ b/nova/tests/api/test.py @@ -0,0 +1,81 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Test for the root WSGI middleware for all API controllers. +""" + +import unittest + +import stubout +import webob +import webob.dec + +import nova.exception +from nova import api +from nova.tests.api.fakes import APIStub + + +class Test(unittest.TestCase): + + def setUp(self): + self.stubs = stubout.StubOutForTesting() + + def tearDown(self): + self.stubs.UnsetAll() + + def _request(self, url, subdomain, **kwargs): + environ_keys = {'HTTP_HOST': '%s.example.com' % subdomain} + environ_keys.update(kwargs) + req = webob.Request.blank(url, environ_keys) + return req.get_response(api.API('ec2')) + + def test_openstack(self): + self.stubs.Set(api.openstack, 'API', APIStub) + result = self._request('/v1.0/cloud', 'api') + self.assertEqual(result.body, "/cloud") + + def test_ec2(self): + self.stubs.Set(api.ec2, 'API', APIStub) + result = self._request('/services/cloud', 'ec2') + self.assertEqual(result.body, "/cloud") + + def test_not_found(self): + self.stubs.Set(api.ec2, 'API', APIStub) + self.stubs.Set(api.openstack, 'API', APIStub) + result = self._request('/test/cloud', 'ec2') + self.assertNotEqual(result.body, "/cloud") + + def test_query_api_versions(self): + result = self._request('/', 'api') + self.assertTrue('CURRENT' in result.body) + + def test_metadata(self): + def go(url): + result = self._request(url, 'ec2', REMOTE_ADDR='128.192.151.2') + # Each should get to the ORM layer and fail to find the IP + self.assertRaises(nova.exception.NotFound, go, '/latest/') + self.assertRaises(nova.exception.NotFound, go, '/2009-04-04/') + self.assertRaises(nova.exception.NotFound, go, '/1.0/') + + def test_ec2_root(self): + result = self._request('/', 'ec2') + self.assertTrue('2007-12-15\n' in result.body) + + +if __name__ == '__main__': + unittest.main() diff --git a/nova/tests/api_integration.py b/nova/tests/api_integration.py deleted file mode 100644 index 54403c655..000000000 --- a/nova/tests/api_integration.py +++ /dev/null @@ -1,54 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import boto -from boto.ec2.regioninfo import RegionInfo -import unittest - - -ACCESS_KEY = 'fake' -SECRET_KEY = 'fake' -CLC_IP = '127.0.0.1' -CLC_PORT = 8773 -REGION = 'test' - - -def get_connection(): - return boto.connect_ec2( - aws_access_key_id=ACCESS_KEY, - aws_secret_access_key=SECRET_KEY, - is_secure=False, - region=RegionInfo(None, REGION, CLC_IP), - port=CLC_PORT, - path='/services/Cloud', - debug=99) - - -class APIIntegrationTests(unittest.TestCase): - def test_001_get_all_images(self): - conn = get_connection() - res = conn.get_all_images() - - -if __name__ == '__main__': - unittest.main() - -#print conn.get_all_key_pairs() -#print conn.create_key_pair -#print conn.create_security_group('name', 'description') diff --git a/nova/tests/db/__init__.py b/nova/tests/db/__init__.py new file mode 100644 index 000000000..2d43aac42 --- /dev/null +++ b/nova/tests/db/__init__.py @@ -0,0 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +:mod:`db` -- Stubs for DB API +============================= +""" diff --git a/nova/tests/db/fakes.py b/nova/tests/db/fakes.py new file mode 100644 index 000000000..05bdd172e --- /dev/null +++ b/nova/tests/db/fakes.py @@ -0,0 +1,75 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack, LLC +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Stubouts, mocks and fixtures for the test suite""" + +import time + +from nova import db +from nova import utils +from nova.compute import instance_types + + +def stub_out_db_instance_api(stubs): + """ Stubs out the db API for creating Instances """ + + class FakeModel(object): + """ Stubs out for model """ + def __init__(self, values): + self.values = values + + def __getattr__(self, name): + return self.values[name] + + def __getitem__(self, key): + if key in self.values: + return self.values[key] + else: + raise NotImplementedError() + + def fake_instance_create(values): + """ Stubs out the db.instance_create method """ + + type_data = instance_types.INSTANCE_TYPES[values['instance_type']] + + base_options = { + 'name': values['name'], + 'id': values['id'], + 'reservation_id': utils.generate_uid('r'), + 'image_id': values['image_id'], + 'kernel_id': values['kernel_id'], + 'ramdisk_id': values['ramdisk_id'], + 'state_description': 'scheduling', + 'user_id': values['user_id'], + 'project_id': values['project_id'], + 'launch_time': time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()), + 'instance_type': values['instance_type'], + 'memory_mb': type_data['memory_mb'], + 'mac_address': values['mac_address'], + 'vcpus': type_data['vcpus'], + 'local_gb': type_data['local_gb'], + } + return FakeModel(base_options) + + def fake_network_get_by_instance(context, instance_id): + fields = { + 'bridge': 'xenbr0', + } + return FakeModel(fields) + + stubs.Set(db, 'instance_create', fake_instance_create) + stubs.Set(db, 'network_get_by_instance', fake_network_get_by_instance) diff --git a/nova/tests/misc_unittest.py b/nova/tests/misc_unittest.py deleted file mode 100644 index 856060afa..000000000 --- a/nova/tests/misc_unittest.py +++ /dev/null @@ -1,48 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os -import subprocess - -from nova import test -from nova.utils import parse_mailmap, str_dict_replace - - -class ProjectTestCase(test.TrialTestCase): - def test_authors_up_to_date(self): - if os.path.exists('../.bzr'): - log_cmd = subprocess.Popen(["bzr", "log", "-n0"], - stdout=subprocess.PIPE) - changelog = log_cmd.communicate()[0] - mailmap = parse_mailmap('../.mailmap') - - contributors = set() - for l in changelog.split('\n'): - l = l.strip() - if (l.startswith('author:') or l.startswith('committer:') - and not l == 'committer: Tarmac'): - email = l.split(' ')[-1] - contributors.add(str_dict_replace(email, mailmap)) - - authors_file = open('../Authors', 'r').read() - - missing = set() - for contributor in contributors: - if not contributor in authors_file: - missing.add(contributor) - - self.assertTrue(len(missing) == 0, - '%r not listed in Authors' % missing) diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index 061799923..ceac17adb 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -54,7 +54,7 @@ os.makedirs(os.path.join(OSS_TEMPDIR, 'images')) os.makedirs(os.path.join(OSS_TEMPDIR, 'buckets')) -class ObjectStoreTestCase(test.TrialTestCase): +class ObjectStoreTestCase(test.TestCase): """Test objectstore API directly.""" def setUp(self): @@ -191,7 +191,7 @@ class TestSite(server.Site): protocol = TestHTTPChannel -class S3APITestCase(test.TrialTestCase): +class S3APITestCase(test.TestCase): """Test objectstore through S3 API.""" def setUp(self): diff --git a/nova/tests/process_unittest.py b/nova/tests/process_unittest.py deleted file mode 100644 index 67245af03..000000000 --- a/nova/tests/process_unittest.py +++ /dev/null @@ -1,132 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import logging -from twisted.internet import defer -from twisted.internet import reactor -from xml.etree import ElementTree - -from nova import exception -from nova import flags -from nova import process -from nova import test -from nova import utils - -FLAGS = flags.FLAGS - - -class ProcessTestCase(test.TrialTestCase): - def setUp(self): - logging.getLogger().setLevel(logging.DEBUG) - super(ProcessTestCase, self).setUp() - - def test_execute_stdout(self): - pool = process.ProcessPool(2) - d = pool.simple_execute('echo test') - - def _check(rv): - self.assertEqual(rv[0], 'test\n') - self.assertEqual(rv[1], '') - - d.addCallback(_check) - d.addErrback(self.fail) - return d - - def test_execute_stderr(self): - pool = process.ProcessPool(2) - d = pool.simple_execute('cat BAD_FILE', check_exit_code=False) - - def _check(rv): - self.assertEqual(rv[0], '') - self.assert_('No such file' in rv[1]) - - d.addCallback(_check) - d.addErrback(self.fail) - return d - - def test_execute_unexpected_stderr(self): - pool = process.ProcessPool(2) - d = pool.simple_execute('cat BAD_FILE') - d.addCallback(lambda x: self.fail('should have raised an error')) - d.addErrback(lambda failure: failure.trap(IOError)) - return d - - def test_max_processes(self): - pool = process.ProcessPool(2) - d1 = pool.simple_execute('sleep 0.01') - d2 = pool.simple_execute('sleep 0.01') - d3 = pool.simple_execute('sleep 0.005') - d4 = pool.simple_execute('sleep 0.005') - - called = [] - - def _called(rv, name): - called.append(name) - - d1.addCallback(_called, 'd1') - d2.addCallback(_called, 'd2') - d3.addCallback(_called, 'd3') - d4.addCallback(_called, 'd4') - - # Make sure that d3 and d4 had to wait on the other two and were called - # in order - # NOTE(termie): there may be a race condition in this test if for some - # reason one of the sleeps takes longer to complete - # than it should - d4.addCallback(lambda x: self.assertEqual(called[2], 'd3')) - d4.addCallback(lambda x: self.assertEqual(called[3], 'd4')) - d4.addErrback(self.fail) - return d4 - - def test_kill_long_process(self): - pool = process.ProcessPool(2) - - d1 = pool.simple_execute('sleep 1') - d2 = pool.simple_execute('sleep 0.005') - - timeout = reactor.callLater(0.1, self.fail, 'should have been killed') - - # kill d1 and wait on it to end then cancel the timeout - d2.addCallback(lambda _: d1.process.signalProcess('KILL')) - d2.addCallback(lambda _: d1) - d2.addBoth(lambda _: timeout.active() and timeout.cancel()) - d2.addErrback(self.fail) - return d2 - - def test_process_exit_is_contained(self): - pool = process.ProcessPool(2) - - d1 = pool.simple_execute('sleep 1') - d1.addCallback(lambda x: self.fail('should have errbacked')) - d1.addErrback(lambda fail: fail.trap(IOError)) - reactor.callLater(0.05, d1.process.signalProcess, 'KILL') - - return d1 - - def test_shared_pool_is_singleton(self): - pool1 = process.SharedPool() - pool2 = process.SharedPool() - self.assertEqual(id(pool1._instance), id(pool2._instance)) - - def test_shared_pool_works_as_singleton(self): - d1 = process.simple_execute('sleep 1') - d2 = process.simple_execute('sleep 0.005') - # lp609749: would have failed with - # exceptions.AssertionError: Someone released me too many times: - # too many tokens! - return d1 diff --git a/nova/tests/access_unittest.py b/nova/tests/test_access.py index 0f66c0a26..58fdea3b5 100644 --- a/nova/tests/access_unittest.py +++ b/nova/tests/test_access.py @@ -35,7 +35,7 @@ class Context(object): pass -class AccessTestCase(test.TrialTestCase): +class AccessTestCase(test.TestCase): def setUp(self): super(AccessTestCase, self).setUp() um = manager.AuthManager() diff --git a/nova/tests/api_unittest.py b/nova/tests/test_api.py index 33d4cb294..33d4cb294 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/test_api.py diff --git a/nova/tests/auth_unittest.py b/nova/tests/test_auth.py index fe891beee..15d40bc53 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/test_auth.py @@ -208,17 +208,13 @@ class AuthManagerTestCase(object): # so it probably belongs in crypto_unittest # but I'm leaving it where I found it. with user_and_project_generator(self.manager) as (user, project): - # NOTE(todd): Should mention why we must setup controller first - # (somebody please clue me in) - cloud_controller = cloud.CloudController() - cloud_controller.setup() - _key, cert_str = self.manager._generate_x509_cert('test1', - 'testproj') + # NOTE(vish): Setup runs genroot.sh if it hasn't been run + cloud.CloudController().setup() + _key, cert_str = crypto.generate_x509_cert(user.id, project.id) logging.debug(cert_str) - # Need to verify that it's signed by the right intermediate CA - full_chain = crypto.fetch_ca(project_id='testproj', chain=True) - int_cert = crypto.fetch_ca(project_id='testproj', chain=False) + full_chain = crypto.fetch_ca(project_id=project.id, chain=True) + int_cert = crypto.fetch_ca(project_id=project.id, chain=False) cloud_cert = crypto.fetch_ca() logging.debug("CA chain:\n\n =====\n%s\n\n=====" % full_chain) signed_cert = X509.load_cert_string(cert_str) @@ -227,7 +223,8 @@ class AuthManagerTestCase(object): cloud_cert = X509.load_cert_string(cloud_cert) self.assertTrue(signed_cert.verify(chain_cert.get_pubkey())) self.assertTrue(signed_cert.verify(int_cert.get_pubkey())) - if not FLAGS.use_intermediate_ca: + + if not FLAGS.use_project_ca: self.assertTrue(signed_cert.verify(cloud_cert.get_pubkey())) else: self.assertFalse(signed_cert.verify(cloud_cert.get_pubkey())) @@ -326,24 +323,20 @@ class AuthManagerTestCase(object): self.assertTrue(user.is_admin()) -class AuthManagerLdapTestCase(AuthManagerTestCase, test.TrialTestCase): +class AuthManagerLdapTestCase(AuthManagerTestCase, test.TestCase): auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' def __init__(self, *args, **kwargs): AuthManagerTestCase.__init__(self) - test.TrialTestCase.__init__(self, *args, **kwargs) + test.TestCase.__init__(self, *args, **kwargs) import nova.auth.fakeldap as fakeldap - FLAGS.redis_db = 8 if FLAGS.flush_db: - logging.info("Flushing redis datastore") - try: - r = fakeldap.Redis.instance() - r.flushdb() - except: - self.skip = True + logging.info("Flushing datastore") + r = fakeldap.Store.instance() + r.flushdb() -class AuthManagerDbTestCase(AuthManagerTestCase, test.TrialTestCase): +class AuthManagerDbTestCase(AuthManagerTestCase, test.TestCase): auth_driver = 'nova.auth.dbdriver.DbDriver' diff --git a/nova/tests/cloud_unittest.py b/nova/tests/test_cloud.py index 9886a2449..70d2c44da 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/test_cloud.py @@ -22,22 +22,18 @@ import logging from M2Crypto import BIO from M2Crypto import RSA import os -import StringIO import tempfile import time from eventlet import greenthread -from twisted.internet import defer -import unittest -from xml.etree import ElementTree from nova import context from nova import crypto from nova import db from nova import flags from nova import rpc +from nova import service from nova import test -from nova import utils from nova.auth import manager from nova.compute import power_state from nova.api.ec2 import cloud @@ -53,10 +49,11 @@ IMAGES_PATH = os.path.join(OSS_TEMPDIR, 'images') os.makedirs(IMAGES_PATH) -class CloudTestCase(test.TrialTestCase): +class CloudTestCase(test.TestCase): def setUp(self): super(CloudTestCase, self).setUp() - self.flags(connection_type='fake', images_path=IMAGES_PATH) + self.flags(connection_type='fake', + images_path=IMAGES_PATH) self.conn = rpc.Connection.instance() logging.getLogger().setLevel(logging.DEBUG) @@ -64,27 +61,23 @@ class CloudTestCase(test.TrialTestCase): # set up our cloud self.cloud = cloud.CloudController() - # set up a service - self.compute = utils.import_object(FLAGS.compute_manager) - self.compute_consumer = rpc.AdapterConsumer(connection=self.conn, - topic=FLAGS.compute_topic, - proxy=self.compute) - self.compute_consumer.attach_to_eventlet() - self.network = utils.import_object(FLAGS.network_manager) - self.network_consumer = rpc.AdapterConsumer(connection=self.conn, - topic=FLAGS.network_topic, - proxy=self.network) - self.network_consumer.attach_to_eventlet() + # set up services + self.compute = service.Service.create(binary='nova-compute') + self.compute.start() + self.network = service.Service.create(binary='nova-network') + self.network.start() self.manager = manager.AuthManager() self.user = self.manager.create_user('admin', 'admin', 'admin', True) self.project = self.manager.create_project('proj', 'admin', 'proj') self.context = context.RequestContext(user=self.user, - project=self.project) + project=self.project) def tearDown(self): self.manager.delete_project(self.project) self.manager.delete_user(self.user) + self.compute.kill() + self.network.kill() super(CloudTestCase, self).tearDown() def _create_key(self, name): @@ -111,12 +104,13 @@ class CloudTestCase(test.TrialTestCase): {'address': address, 'host': FLAGS.host}) self.cloud.allocate_address(self.context) - inst = db.instance_create(self.context, {}) + inst = db.instance_create(self.context, {'host': FLAGS.host}) fixed = self.network.allocate_fixed_ip(self.context, inst['id']) ec2_id = cloud.internal_id_to_ec2_id(inst['internal_id']) self.cloud.associate_address(self.context, instance_id=ec2_id, public_ip=address) + greenthread.sleep(0.3) self.cloud.disassociate_address(self.context, public_ip=address) self.cloud.release_address(self.context, @@ -126,6 +120,19 @@ class CloudTestCase(test.TrialTestCase): db.instance_destroy(self.context, inst['id']) db.floating_ip_destroy(self.context, address) + def test_describe_volumes(self): + """Makes sure describe_volumes works and filters results.""" + vol1 = db.volume_create(self.context, {}) + vol2 = db.volume_create(self.context, {}) + result = self.cloud.describe_volumes(self.context) + self.assertEqual(len(result['volumeSet']), 2) + result = self.cloud.describe_volumes(self.context, + volume_id=[vol2['ec2_id']]) + self.assertEqual(len(result['volumeSet']), 1) + self.assertEqual(result['volumeSet'][0]['volumeId'], vol2['ec2_id']) + db.volume_destroy(self.context, vol1['id']) + db.volume_destroy(self.context, vol2['id']) + def test_console_output(self): image_id = FLAGS.default_image instance_type = FLAGS.default_instance_type @@ -186,7 +193,7 @@ class CloudTestCase(test.TrialTestCase): logging.debug("Need to watch instance %s until it's running..." % instance['instance_id']) while True: - rv = yield defer.succeed(time.sleep(1)) + greenthread.sleep(1) info = self.cloud._get_instance(instance['instance_id']) logging.debug(info['state']) if info['state'] == power_state.RUNNING: diff --git a/nova/tests/compute_unittest.py b/nova/tests/test_compute.py index a55449739..348bb3351 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/test_compute.py @@ -22,8 +22,6 @@ Tests For Compute import datetime import logging -from twisted.internet import defer - from nova import context from nova import db from nova import exception @@ -33,15 +31,17 @@ from nova import utils from nova.auth import manager from nova.compute import api as compute_api + FLAGS = flags.FLAGS -class ComputeTestCase(test.TrialTestCase): +class ComputeTestCase(test.TestCase): """Test case for compute""" def setUp(self): logging.getLogger().setLevel(logging.DEBUG) super(ComputeTestCase, self).setUp() self.flags(connection_type='fake', + stub_network=True, network_manager='nova.network.manager.FlatManager') self.compute = utils.import_object(FLAGS.compute_manager) self.compute_api = compute_api.ComputeAPI() @@ -72,52 +72,44 @@ class ComputeTestCase(test.TrialTestCase): """Verify that an instance cannot be created without a display_name.""" cases = [dict(), dict(display_name=None)] for instance in cases: - ref = self.compute_api.create_instance(self.context, None, - **instance) + ref = self.compute_api.create_instances(self.context, + FLAGS.default_instance_type, None, **instance) try: - self.assertNotEqual(ref.display_name, None) + self.assertNotEqual(ref[0].display_name, None) finally: - db.instance_destroy(self.context, ref['id']) + db.instance_destroy(self.context, ref[0]['id']) def test_create_instance_associates_security_groups(self): - """Make sure create_instance associates security groups""" - inst = {} - inst['user_id'] = self.user.id - inst['project_id'] = self.project.id + """Make sure create_instances associates security groups""" values = {'name': 'default', 'description': 'default', 'user_id': self.user.id, 'project_id': self.project.id} group = db.security_group_create(self.context, values) - ref = self.compute_api.create_instance(self.context, - security_groups=[group['id']], - **inst) - # reload to get groups - instance_ref = db.instance_get(self.context, ref['id']) + ref = self.compute_api.create_instances(self.context, + FLAGS.default_instance_type, None, security_group=['default']) try: - self.assertEqual(len(instance_ref['security_groups']), 1) + self.assertEqual(len(ref[0]['security_groups']), 1) finally: db.security_group_destroy(self.context, group['id']) - db.instance_destroy(self.context, instance_ref['id']) + db.instance_destroy(self.context, ref[0]['id']) - @defer.inlineCallbacks def test_run_terminate(self): """Make sure it is possible to run and terminate instance""" instance_id = self._create_instance() - yield self.compute.run_instance(self.context, instance_id) + self.compute.run_instance(self.context, instance_id) instances = db.instance_get_all(context.get_admin_context()) logging.info("Running instances: %s", instances) self.assertEqual(len(instances), 1) - yield self.compute.terminate_instance(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) instances = db.instance_get_all(context.get_admin_context()) logging.info("After terminating instances: %s", instances) self.assertEqual(len(instances), 0) - @defer.inlineCallbacks def test_run_terminate_timestamps(self): """Make sure timestamps are set for launched and destroyed""" instance_id = self._create_instance() @@ -125,42 +117,48 @@ class ComputeTestCase(test.TrialTestCase): self.assertEqual(instance_ref['launched_at'], None) self.assertEqual(instance_ref['deleted_at'], None) launch = datetime.datetime.utcnow() - yield self.compute.run_instance(self.context, instance_id) + self.compute.run_instance(self.context, instance_id) instance_ref = db.instance_get(self.context, instance_id) self.assert_(instance_ref['launched_at'] > launch) self.assertEqual(instance_ref['deleted_at'], None) terminate = datetime.datetime.utcnow() - yield self.compute.terminate_instance(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) self.context = self.context.elevated(True) instance_ref = db.instance_get(self.context, instance_id) self.assert_(instance_ref['launched_at'] < terminate) self.assert_(instance_ref['deleted_at'] > terminate) - @defer.inlineCallbacks + def test_pause(self): + """Ensure instance can be paused""" + instance_id = self._create_instance() + self.compute.run_instance(self.context, instance_id) + self.compute.pause_instance(self.context, instance_id) + self.compute.unpause_instance(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) + def test_reboot(self): """Ensure instance can be rebooted""" instance_id = self._create_instance() - yield self.compute.run_instance(self.context, instance_id) - yield self.compute.reboot_instance(self.context, instance_id) - yield self.compute.terminate_instance(self.context, instance_id) + self.compute.run_instance(self.context, instance_id) + self.compute.reboot_instance(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) - @defer.inlineCallbacks def test_console_output(self): """Make sure we can get console output from instance""" instance_id = self._create_instance() - yield self.compute.run_instance(self.context, instance_id) + self.compute.run_instance(self.context, instance_id) - console = yield self.compute.get_console_output(self.context, + console = self.compute.get_console_output(self.context, instance_id) self.assert_(console) - yield self.compute.terminate_instance(self.context, instance_id) + self.compute.terminate_instance(self.context, instance_id) - @defer.inlineCallbacks def test_run_instance_existing(self): """Ensure failure when running an instance that already exists""" instance_id = self._create_instance() - yield self.compute.run_instance(self.context, instance_id) - self.assertFailure(self.compute.run_instance(self.context, - instance_id), - exception.Error) - yield self.compute.terminate_instance(self.context, instance_id) + self.compute.run_instance(self.context, instance_id) + self.assertRaises(exception.Error, + self.compute.run_instance, + self.context, + instance_id) + self.compute.terminate_instance(self.context, instance_id) diff --git a/nova/tests/flags_unittest.py b/nova/tests/test_flags.py index b97df075d..707300fcf 100644 --- a/nova/tests/flags_unittest.py +++ b/nova/tests/test_flags.py @@ -24,7 +24,7 @@ FLAGS = flags.FLAGS flags.DEFINE_string('flags_unittest', 'foo', 'for testing purposes only') -class FlagsTestCase(test.TrialTestCase): +class FlagsTestCase(test.TestCase): def setUp(self): super(FlagsTestCase, self).setUp() diff --git a/nova/tests/test_middleware.py b/nova/tests/test_middleware.py new file mode 100644 index 000000000..0febf52d6 --- /dev/null +++ b/nova/tests/test_middleware.py @@ -0,0 +1,86 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import datetime +import webob +import webob.dec +import webob.exc + +from nova.api import ec2 +from nova import flags +from nova import test +from nova import utils + + +FLAGS = flags.FLAGS + + +@webob.dec.wsgify +def conditional_forbid(req): + """Helper wsgi app returns 403 if param 'die' is 1.""" + if 'die' in req.params and req.params['die'] == '1': + raise webob.exc.HTTPForbidden() + return 'OK' + + +class LockoutTestCase(test.TrialTestCase): + """Test case for the Lockout middleware.""" + def setUp(self): # pylint: disable-msg=C0103 + super(LockoutTestCase, self).setUp() + utils.set_time_override() + self.lockout = ec2.Lockout(conditional_forbid) + + def tearDown(self): # pylint: disable-msg=C0103 + utils.clear_time_override() + super(LockoutTestCase, self).tearDown() + + def _send_bad_attempts(self, access_key, num_attempts=1): + """Fail x.""" + for i in xrange(num_attempts): + req = webob.Request.blank('/?AWSAccessKeyId=%s&die=1' % access_key) + self.assertEqual(req.get_response(self.lockout).status_int, 403) + + def _is_locked_out(self, access_key): + """Sends a test request to see if key is locked out.""" + req = webob.Request.blank('/?AWSAccessKeyId=%s' % access_key) + return (req.get_response(self.lockout).status_int == 403) + + def test_lockout(self): + self._send_bad_attempts('test', FLAGS.lockout_attempts) + self.assertTrue(self._is_locked_out('test')) + + def test_timeout(self): + self._send_bad_attempts('test', FLAGS.lockout_attempts) + self.assertTrue(self._is_locked_out('test')) + utils.advance_time_seconds(FLAGS.lockout_minutes * 60) + self.assertFalse(self._is_locked_out('test')) + + def test_multiple_keys(self): + self._send_bad_attempts('test1', FLAGS.lockout_attempts) + self.assertTrue(self._is_locked_out('test1')) + self.assertFalse(self._is_locked_out('test2')) + utils.advance_time_seconds(FLAGS.lockout_minutes * 60) + self.assertFalse(self._is_locked_out('test1')) + self.assertFalse(self._is_locked_out('test2')) + + def test_window_timeout(self): + self._send_bad_attempts('test', FLAGS.lockout_attempts - 1) + self.assertFalse(self._is_locked_out('test')) + utils.advance_time_seconds(FLAGS.lockout_window * 60) + self._send_bad_attempts('test', FLAGS.lockout_attempts - 1) + self.assertFalse(self._is_locked_out('test')) diff --git a/nova/tests/test_misc.py b/nova/tests/test_misc.py new file mode 100644 index 000000000..33c1777d5 --- /dev/null +++ b/nova/tests/test_misc.py @@ -0,0 +1,55 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from nova import test +from nova.utils import parse_mailmap, str_dict_replace + + +class ProjectTestCase(test.TestCase): + def test_authors_up_to_date(self): + if os.path.exists('.bzr'): + contributors = set() + + mailmap = parse_mailmap('.mailmap') + + import bzrlib.workingtree + tree = bzrlib.workingtree.WorkingTree.open('.') + tree.lock_read() + try: + parents = tree.get_parent_ids() + g = tree.branch.repository.get_graph() + for p in parents[1:]: + rev_ids = [r for r, _ in g.iter_ancestry(parents) + if r != "null:"] + revs = tree.branch.repository.get_revisions(rev_ids) + for r in revs: + for author in r.get_apparent_authors(): + email = author.split(' ')[-1] + contributors.add(str_dict_replace(email, mailmap)) + + authors_file = open('Authors', 'r').read() + + missing = set() + for contributor in contributors: + if not contributor in authors_file: + missing.add(contributor) + + self.assertTrue(len(missing) == 0, + '%r not listed in Authors' % missing) + finally: + tree.unlock() diff --git a/nova/tests/network_unittest.py b/nova/tests/test_network.py index 6f4705719..96473ac7c 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/test_network.py @@ -26,6 +26,7 @@ from nova import context from nova import db from nova import exception from nova import flags +from nova import service from nova import test from nova import utils from nova.auth import manager @@ -33,13 +34,14 @@ from nova.auth import manager FLAGS = flags.FLAGS -class NetworkTestCase(test.TrialTestCase): +class NetworkTestCase(test.TestCase): """Test cases for network code""" def setUp(self): super(NetworkTestCase, self).setUp() # NOTE(vish): if you change these flags, make sure to change the # flags in the corresponding section in nova-dhcpbridge self.flags(connection_type='fake', + fake_call=True, fake_network=True, network_size=16, num_networks=5) @@ -56,16 +58,13 @@ class NetworkTestCase(test.TrialTestCase): # create the necessary network data for the project user_context = context.RequestContext(project=self.projects[i], user=self.user) - network_ref = self.network.get_network(user_context) - self.network.set_network_host(context.get_admin_context(), - network_ref['id']) + host = self.network.get_network_host(user_context.elevated()) instance_ref = self._create_instance(0) self.instance_id = instance_ref['id'] instance_ref = self._create_instance(1) self.instance2_id = instance_ref['id'] def tearDown(self): - super(NetworkTestCase, self).tearDown() # TODO(termie): this should really be instantiating clean datastores # in between runs, one failure kills all the tests db.instance_destroy(context.get_admin_context(), self.instance_id) @@ -73,6 +72,7 @@ class NetworkTestCase(test.TrialTestCase): for project in self.projects: self.manager.delete_project(project) self.manager.delete_user(self.user) + super(NetworkTestCase, self).tearDown() def _create_instance(self, project_num, mac=None): if not mac: diff --git a/nova/tests/quota_unittest.py b/nova/tests/test_quota.py index 1966b51f7..8cf2a5e54 100644 --- a/nova/tests/quota_unittest.py +++ b/nova/tests/test_quota.py @@ -32,7 +32,7 @@ from nova.api.ec2 import cloud FLAGS = flags.FLAGS -class QuotaTestCase(test.TrialTestCase): +class QuotaTestCase(test.TestCase): def setUp(self): logging.getLogger().setLevel(logging.DEBUG) super(QuotaTestCase, self).setUp() diff --git a/nova/tests/rpc_unittest.py b/nova/tests/test_rpc.py index f35b65a39..6ea2edcab 100644 --- a/nova/tests/rpc_unittest.py +++ b/nova/tests/test_rpc.py @@ -20,8 +20,6 @@ Unit Tests for remote procedure calls using queue """ import logging -from twisted.internet import defer - from nova import context from nova import flags from nova import rpc @@ -31,32 +29,31 @@ from nova import test FLAGS = flags.FLAGS -class RpcTestCase(test.TrialTestCase): +class RpcTestCase(test.TestCase): """Test cases for rpc""" def setUp(self): super(RpcTestCase, self).setUp() - self.conn = rpc.Connection.instance() + self.conn = rpc.Connection.instance(True) self.receiver = TestReceiver() self.consumer = rpc.AdapterConsumer(connection=self.conn, topic='test', proxy=self.receiver) - self.consumer.attach_to_twisted() + self.consumer.attach_to_eventlet() self.context = context.get_admin_context() def test_call_succeed(self): """Get a value through rpc call""" value = 42 - result = yield rpc.call_twisted(self.context, - 'test', {"method": "echo", + result = rpc.call(self.context, 'test', {"method": "echo", "args": {"value": value}}) self.assertEqual(value, result) def test_context_passed(self): """Makes sure a context is passed through rpc call""" value = 42 - result = yield rpc.call_twisted(self.context, - 'test', {"method": "context", - "args": {"value": value}}) + result = rpc.call(self.context, + 'test', {"method": "context", + "args": {"value": value}}) self.assertEqual(self.context.to_dict(), result) def test_call_exception(self): @@ -67,18 +64,48 @@ class RpcTestCase(test.TrialTestCase): to an int in the test. """ value = 42 - self.assertFailure(rpc.call_twisted(self.context, 'test', - {"method": "fail", - "args": {"value": value}}), - rpc.RemoteError) + self.assertRaises(rpc.RemoteError, + rpc.call, + self.context, + 'test', + {"method": "fail", + "args": {"value": value}}) try: - yield rpc.call_twisted(self.context, - 'test', {"method": "fail", - "args": {"value": value}}) + rpc.call(self.context, + 'test', + {"method": "fail", + "args": {"value": value}}) self.fail("should have thrown rpc.RemoteError") except rpc.RemoteError as exc: self.assertEqual(int(exc.value), value) + def test_nested_calls(self): + """Test that we can do an rpc.call inside another call""" + class Nested(object): + @staticmethod + def echo(context, queue, value): + """Calls echo in the passed queue""" + logging.debug("Nested received %s, %s", queue, value) + ret = rpc.call(context, + queue, + {"method": "echo", + "args": {"value": value}}) + logging.debug("Nested return %s", ret) + return value + + nested = Nested() + conn = rpc.Connection.instance(True) + consumer = rpc.AdapterConsumer(connection=conn, + topic='nested', + proxy=nested) + consumer.attach_to_eventlet() + value = 42 + result = rpc.call(self.context, + 'nested', {"method": "echo", + "args": {"queue": "test", + "value": value}}) + self.assertEqual(value, result) + class TestReceiver(object): """Simple Proxy class so the consumer has methods to call @@ -89,13 +116,13 @@ class TestReceiver(object): def echo(context, value): """Simply returns whatever value is sent in""" logging.debug("Received %s", value) - return defer.succeed(value) + return value @staticmethod def context(context, value): """Returns dictionary version of context""" logging.debug("Received %s", context) - return defer.succeed(context.to_dict()) + return context.to_dict() @staticmethod def fail(context, value): diff --git a/nova/tests/scheduler_unittest.py b/nova/tests/test_scheduler.py index cb5fe6b9c..91517cc5d 100644 --- a/nova/tests/scheduler_unittest.py +++ b/nova/tests/test_scheduler.py @@ -44,11 +44,11 @@ class TestDriver(driver.Scheduler): return 'named_host' -class SchedulerTestCase(test.TrialTestCase): +class SchedulerTestCase(test.TestCase): """Test case for scheduler""" def setUp(self): super(SchedulerTestCase, self).setUp() - self.flags(scheduler_driver='nova.tests.scheduler_unittest.TestDriver') + self.flags(scheduler_driver='nova.tests.test_scheduler.TestDriver') def test_fallback(self): scheduler = manager.SchedulerManager() @@ -73,11 +73,12 @@ class SchedulerTestCase(test.TrialTestCase): scheduler.named_method(ctxt, 'topic', num=7) -class SimpleDriverTestCase(test.TrialTestCase): +class SimpleDriverTestCase(test.TestCase): """Test case for simple driver""" def setUp(self): super(SimpleDriverTestCase, self).setUp() self.flags(connection_type='fake', + stub_network=True, max_cores=4, max_gigabytes=4, network_manager='nova.network.manager.FlatManager', @@ -122,12 +123,12 @@ class SimpleDriverTestCase(test.TrialTestCase): 'nova-compute', 'compute', FLAGS.compute_manager) - compute1.startService() + compute1.start() compute2 = service.Service('host2', 'nova-compute', 'compute', FLAGS.compute_manager) - compute2.startService() + compute2.start() hosts = self.scheduler.driver.hosts_up(self.context, 'compute') self.assertEqual(len(hosts), 2) compute1.kill() @@ -139,12 +140,12 @@ class SimpleDriverTestCase(test.TrialTestCase): 'nova-compute', 'compute', FLAGS.compute_manager) - compute1.startService() + compute1.start() compute2 = service.Service('host2', 'nova-compute', 'compute', FLAGS.compute_manager) - compute2.startService() + compute2.start() instance_id1 = self._create_instance() compute1.run_instance(self.context, instance_id1) instance_id2 = self._create_instance() @@ -162,12 +163,12 @@ class SimpleDriverTestCase(test.TrialTestCase): 'nova-compute', 'compute', FLAGS.compute_manager) - compute1.startService() + compute1.start() compute2 = service.Service('host2', 'nova-compute', 'compute', FLAGS.compute_manager) - compute2.startService() + compute2.start() instance_ids1 = [] instance_ids2 = [] for index in xrange(FLAGS.max_cores): @@ -195,12 +196,12 @@ class SimpleDriverTestCase(test.TrialTestCase): 'nova-volume', 'volume', FLAGS.volume_manager) - volume1.startService() + volume1.start() volume2 = service.Service('host2', 'nova-volume', 'volume', FLAGS.volume_manager) - volume2.startService() + volume2.start() volume_id1 = self._create_volume() volume1.create_volume(self.context, volume_id1) volume_id2 = self._create_volume() @@ -218,12 +219,12 @@ class SimpleDriverTestCase(test.TrialTestCase): 'nova-volume', 'volume', FLAGS.volume_manager) - volume1.startService() + volume1.start() volume2 = service.Service('host2', 'nova-volume', 'volume', FLAGS.volume_manager) - volume2.startService() + volume2.start() volume_ids1 = [] volume_ids2 = [] for index in xrange(FLAGS.max_gigabytes): diff --git a/nova/tests/service_unittest.py b/nova/tests/test_service.py index a268bc4fe..b30838ad7 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/test_service.py @@ -22,9 +22,6 @@ Unit Tests for remote procedure calls using queue import mox -from twisted.application.app import startApplication -from twisted.internet import defer - from nova import exception from nova import flags from nova import rpc @@ -33,7 +30,7 @@ from nova import service from nova import manager FLAGS = flags.FLAGS -flags.DEFINE_string("fake_manager", "nova.tests.service_unittest.FakeManager", +flags.DEFINE_string("fake_manager", "nova.tests.test_service.FakeManager", "Manager for testing") @@ -48,34 +45,34 @@ class ExtendedService(service.Service): return 'service' -class ServiceManagerTestCase(test.TrialTestCase): +class ServiceManagerTestCase(test.TestCase): """Test cases for Services""" def test_attribute_error_for_no_manager(self): serv = service.Service('test', 'test', 'test', - 'nova.tests.service_unittest.FakeManager') + 'nova.tests.test_service.FakeManager') self.assertRaises(AttributeError, getattr, serv, 'test_method') def test_message_gets_to_manager(self): serv = service.Service('test', 'test', 'test', - 'nova.tests.service_unittest.FakeManager') - serv.startService() + 'nova.tests.test_service.FakeManager') + serv.start() self.assertEqual(serv.test_method(), 'manager') def test_override_manager_method(self): serv = ExtendedService('test', 'test', 'test', - 'nova.tests.service_unittest.FakeManager') - serv.startService() + 'nova.tests.test_service.FakeManager') + serv.start() self.assertEqual(serv.test_method(), 'service') -class ServiceTestCase(test.TrialTestCase): +class ServiceTestCase(test.TestCase): """Test cases for Services""" def setUp(self): @@ -94,8 +91,6 @@ class ServiceTestCase(test.TrialTestCase): self.mox.StubOutWithMock(rpc, 'AdapterConsumer', use_mock_anything=True) - self.mox.StubOutWithMock( - service.task, 'LoopingCall', use_mock_anything=True) rpc.AdapterConsumer(connection=mox.IgnoreArg(), topic=topic, proxy=mox.IsA(service.Service)).AndReturn( @@ -106,19 +101,8 @@ class ServiceTestCase(test.TrialTestCase): proxy=mox.IsA(service.Service)).AndReturn( rpc.AdapterConsumer) - rpc.AdapterConsumer.attach_to_twisted() - rpc.AdapterConsumer.attach_to_twisted() - - # Stub out looping call a bit needlessly since we don't have an easy - # way to cancel it (yet) when the tests finishes - service.task.LoopingCall(mox.IgnoreArg()).AndReturn( - service.task.LoopingCall) - service.task.LoopingCall.start(interval=mox.IgnoreArg(), - now=mox.IgnoreArg()) - service.task.LoopingCall(mox.IgnoreArg()).AndReturn( - service.task.LoopingCall) - service.task.LoopingCall.start(interval=mox.IgnoreArg(), - now=mox.IgnoreArg()) + rpc.AdapterConsumer.attach_to_eventlet() + rpc.AdapterConsumer.attach_to_eventlet() service_create = {'host': host, 'binary': binary, @@ -136,14 +120,14 @@ class ServiceTestCase(test.TrialTestCase): service_create).AndReturn(service_ref) self.mox.ReplayAll() - startApplication(app, False) + app.start() + app.stop() self.assert_(app) # We're testing sort of weird behavior in how report_state decides # whether it is disconnected, it looks for a variable on itself called # 'model_disconnected' and report_state doesn't really do much so this # these are mostly just for coverage - @defer.inlineCallbacks def test_report_state_no_service(self): host = 'foo' binary = 'bar' @@ -172,11 +156,10 @@ class ServiceTestCase(test.TrialTestCase): serv = service.Service(host, binary, topic, - 'nova.tests.service_unittest.FakeManager') - serv.startService() - yield serv.report_state() + 'nova.tests.test_service.FakeManager') + serv.start() + serv.report_state() - @defer.inlineCallbacks def test_report_state_newly_disconnected(self): host = 'foo' binary = 'bar' @@ -203,12 +186,11 @@ class ServiceTestCase(test.TrialTestCase): serv = service.Service(host, binary, topic, - 'nova.tests.service_unittest.FakeManager') - serv.startService() - yield serv.report_state() + 'nova.tests.test_service.FakeManager') + serv.start() + serv.report_state() self.assert_(serv.model_disconnected) - @defer.inlineCallbacks def test_report_state_newly_connected(self): host = 'foo' binary = 'bar' @@ -237,9 +219,9 @@ class ServiceTestCase(test.TrialTestCase): serv = service.Service(host, binary, topic, - 'nova.tests.service_unittest.FakeManager') - serv.startService() + 'nova.tests.test_service.FakeManager') + serv.start() serv.model_disconnected = True - yield serv.report_state() + serv.report_state() self.assert_(not serv.model_disconnected) diff --git a/nova/tests/twistd_unittest.py b/nova/tests/test_twistd.py index 75007b9c8..75007b9c8 100644 --- a/nova/tests/twistd_unittest.py +++ b/nova/tests/test_twistd.py diff --git a/nova/tests/virt_unittest.py b/nova/tests/test_virt.py index d49383fb7..8dab8de2f 100644 --- a/nova/tests/virt_unittest.py +++ b/nova/tests/test_virt.py @@ -30,9 +30,10 @@ FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') -class LibvirtConnTestCase(test.TrialTestCase): +class LibvirtConnTestCase(test.TestCase): def setUp(self): super(LibvirtConnTestCase, self).setUp() + self.flags(fake_call=True) self.manager = manager.AuthManager() self.user = self.manager.create_user('fake', 'fake', 'fake', admin=True) @@ -40,33 +41,64 @@ class LibvirtConnTestCase(test.TrialTestCase): self.network = utils.import_object(FLAGS.network_manager) FLAGS.instances_path = '' - def test_get_uri_and_template(self): - ip = '10.11.12.13' - - instance = {'internal_id': 1, - 'memory_kb': '1024000', - 'basepath': '/some/path', - 'bridge_name': 'br100', - 'mac_address': '02:12:34:46:56:67', - 'vcpus': 2, - 'project_id': 'fake', - 'bridge': 'br101', - 'instance_type': 'm1.small'} - + test_ip = '10.11.12.13' + test_instance = {'memory_kb': '1024000', + 'basepath': '/some/path', + 'bridge_name': 'br100', + 'mac_address': '02:12:34:46:56:67', + 'vcpus': 2, + 'project_id': 'fake', + 'bridge': 'br101', + 'instance_type': 'm1.small'} + + def test_xml_and_uri_no_ramdisk_no_kernel(self): + instance_data = dict(self.test_instance) + self._check_xml_and_uri(instance_data, + expect_kernel=False, expect_ramdisk=False) + + def test_xml_and_uri_no_ramdisk(self): + instance_data = dict(self.test_instance) + instance_data['kernel_id'] = 'aki-deadbeef' + self._check_xml_and_uri(instance_data, + expect_kernel=True, expect_ramdisk=False) + + def test_xml_and_uri_no_kernel(self): + instance_data = dict(self.test_instance) + instance_data['ramdisk_id'] = 'ari-deadbeef' + self._check_xml_and_uri(instance_data, + expect_kernel=False, expect_ramdisk=False) + + def test_xml_and_uri(self): + instance_data = dict(self.test_instance) + instance_data['ramdisk_id'] = 'ari-deadbeef' + instance_data['kernel_id'] = 'aki-deadbeef' + self._check_xml_and_uri(instance_data, + expect_kernel=True, expect_ramdisk=True) + + def test_xml_and_uri_rescue(self): + instance_data = dict(self.test_instance) + instance_data['ramdisk_id'] = 'ari-deadbeef' + instance_data['kernel_id'] = 'aki-deadbeef' + self._check_xml_and_uri(instance_data, expect_kernel=True, + expect_ramdisk=True, rescue=True) + + def _check_xml_and_uri(self, instance, expect_ramdisk, expect_kernel, + rescue=False): user_context = context.RequestContext(project=self.project, user=self.user) instance_ref = db.instance_create(user_context, instance) - network_ref = self.network.get_network(user_context) - self.network.set_network_host(context.get_admin_context(), - network_ref['id']) + host = self.network.get_network_host(user_context.elevated()) + network_ref = db.project_get_network(context.get_admin_context(), + self.project.id) - fixed_ip = {'address': ip, + fixed_ip = {'address': self.test_ip, 'network_id': network_ref['id']} ctxt = context.get_admin_context() fixed_ip_ref = db.fixed_ip_create(ctxt, fixed_ip) - db.fixed_ip_update(ctxt, ip, {'allocated': True, - 'instance_id': instance_ref['id']}) + db.fixed_ip_update(ctxt, self.test_ip, + {'allocated': True, + 'instance_id': instance_ref['id']}) type_uri_map = {'qemu': ('qemu:///system', [(lambda t: t.find('.').get('type'), 'qemu'), @@ -78,23 +110,73 @@ class LibvirtConnTestCase(test.TrialTestCase): (lambda t: t.find('./devices/emulator'), None)]), 'uml': ('uml:///system', [(lambda t: t.find('.').get('type'), 'uml'), - (lambda t: t.find('./os/type').text, 'uml')])} + (lambda t: t.find('./os/type').text, 'uml')]), + 'xen': ('xen:///', + [(lambda t: t.find('.').get('type'), 'xen'), + (lambda t: t.find('./os/type').text, 'linux')]), + } + + for hypervisor_type in ['qemu', 'kvm', 'xen']: + check_list = type_uri_map[hypervisor_type][1] + + if rescue: + check = (lambda t: t.find('./os/kernel').text.split('/')[1], + 'rescue-kernel') + check_list.append(check) + check = (lambda t: t.find('./os/initrd').text.split('/')[1], + 'rescue-ramdisk') + check_list.append(check) + else: + if expect_kernel: + check = (lambda t: t.find('./os/kernel').text.split( + '/')[1], 'kernel') + else: + check = (lambda t: t.find('./os/kernel'), None) + check_list.append(check) + + if expect_ramdisk: + check = (lambda t: t.find('./os/initrd').text.split( + '/')[1], 'ramdisk') + else: + check = (lambda t: t.find('./os/initrd'), None) + check_list.append(check) common_checks = [ (lambda t: t.find('.').tag, 'domain'), - (lambda t: t.find('./devices/interface/filterref/parameter').\ - get('name'), 'IP'), - (lambda t: t.find('./devices/interface/filterref/parameter').\ - get('value'), '10.11.12.13')] + (lambda t: t.find( + './devices/interface/filterref/parameter').get('name'), 'IP'), + (lambda t: t.find( + './devices/interface/filterref/parameter').get( + 'value'), '10.11.12.13'), + (lambda t: t.findall( + './devices/interface/filterref/parameter')[1].get( + 'name'), 'DHCPSERVER'), + (lambda t: t.findall( + './devices/interface/filterref/parameter')[1].get( + 'value'), '10.0.0.1'), + (lambda t: t.find('./devices/serial/source').get( + 'path').split('/')[1], 'console.log'), + (lambda t: t.find('./memory').text, '2097152')] + + if rescue: + common_checks += [ + (lambda t: t.findall('./devices/disk/source')[0].get( + 'file').split('/')[1], 'rescue-disk'), + (lambda t: t.findall('./devices/disk/source')[1].get( + 'file').split('/')[1], 'disk')] + else: + common_checks += [(lambda t: t.findall( + './devices/disk/source')[0].get('file').split('/')[1], + 'disk')] for (libvirt_type, (expected_uri, checks)) in type_uri_map.iteritems(): FLAGS.libvirt_type = libvirt_type conn = libvirt_conn.LibvirtConnection(True) - uri, _template, _rescue = conn.get_uri_and_templates() + uri = conn.get_uri() self.assertEquals(uri, expected_uri) - xml = conn.to_xml(instance_ref) + xml = conn.to_xml(instance_ref, rescue) tree = xml_to_tree(xml) for i, (check, expected_result) in enumerate(checks): self.assertEqual(check(tree), @@ -106,6 +188,9 @@ class LibvirtConnTestCase(test.TrialTestCase): expected_result, '%s failed common check %d' % (xml, i)) + # This test is supposed to make sure we don't override a specifically + # set uri + # # Deliberately not just assigning this string to FLAGS.libvirt_uri and # checking against that later on. This way we make sure the # implementation doesn't fiddle around with the FLAGS. @@ -114,7 +199,7 @@ class LibvirtConnTestCase(test.TrialTestCase): for (libvirt_type, (expected_uri, checks)) in type_uri_map.iteritems(): FLAGS.libvirt_type = libvirt_type conn = libvirt_conn.LibvirtConnection(True) - uri, _template, _rescue = conn.get_uri_and_templates() + uri = conn.get_uri() self.assertEquals(uri, testuri) def tearDown(self): @@ -123,7 +208,7 @@ class LibvirtConnTestCase(test.TrialTestCase): self.manager.delete_user(self.user) -class NWFilterTestCase(test.TrialTestCase): +class NWFilterTestCase(test.TestCase): def setUp(self): super(NWFilterTestCase, self).setUp() @@ -235,7 +320,7 @@ class NWFilterTestCase(test.TrialTestCase): 'project_id': 'fake'}) inst_id = instance_ref['id'] - def _ensure_all_called(_): + def _ensure_all_called(): instance_filter = 'nova-instance-%s' % instance_ref['name'] secgroup_filter = 'nova-secgroup-%s' % self.security_group['id'] for required in [secgroup_filter, 'allow-dhcp-server', @@ -252,8 +337,7 @@ class NWFilterTestCase(test.TrialTestCase): self.security_group.id) instance = db.instance_get(self.context, inst_id) - d = self.fw.setup_nwfilters_for_instance(instance) - d.addCallback(_ensure_all_called) - d.addCallback(lambda _: self.teardown_security_group()) - - return d + self.fw.setup_base_nwfilters() + self.fw.setup_nwfilters_for_instance(instance) + _ensure_all_called() + self.teardown_security_group() diff --git a/nova/tests/volume_unittest.py b/nova/tests/test_volume.py index 12321a96f..b13455fb0 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/test_volume.py @@ -21,8 +21,6 @@ Tests for Volume Code. """ import logging -from twisted.internet import defer - from nova import context from nova import exception from nova import db @@ -33,7 +31,7 @@ from nova import utils FLAGS = flags.FLAGS -class VolumeTestCase(test.TrialTestCase): +class VolumeTestCase(test.TestCase): """Test Case for volumes.""" def setUp(self): @@ -56,51 +54,48 @@ class VolumeTestCase(test.TrialTestCase): vol['attach_status'] = "detached" return db.volume_create(context.get_admin_context(), vol)['id'] - @defer.inlineCallbacks def test_create_delete_volume(self): """Test volume can be created and deleted.""" volume_id = self._create_volume() - yield self.volume.create_volume(self.context, volume_id) + self.volume.create_volume(self.context, volume_id) self.assertEqual(volume_id, db.volume_get(context.get_admin_context(), volume_id).id) - yield self.volume.delete_volume(self.context, volume_id) + self.volume.delete_volume(self.context, volume_id) self.assertRaises(exception.NotFound, db.volume_get, self.context, volume_id) - @defer.inlineCallbacks def test_too_big_volume(self): """Ensure failure if a too large of a volume is requested.""" # FIXME(vish): validation needs to move into the data layer in # volume_create - defer.returnValue(True) + return True try: volume_id = self._create_volume('1001') - yield self.volume.create_volume(self.context, volume_id) + self.volume.create_volume(self.context, volume_id) self.fail("Should have thrown TypeError") except TypeError: pass - @defer.inlineCallbacks def test_too_many_volumes(self): """Ensure that NoMoreTargets is raised when we run out of volumes.""" vols = [] total_slots = FLAGS.iscsi_num_targets for _index in xrange(total_slots): volume_id = self._create_volume() - yield self.volume.create_volume(self.context, volume_id) + self.volume.create_volume(self.context, volume_id) vols.append(volume_id) volume_id = self._create_volume() - self.assertFailure(self.volume.create_volume(self.context, - volume_id), - db.NoMoreTargets) + self.assertRaises(db.NoMoreTargets, + self.volume.create_volume, + self.context, + volume_id) db.volume_destroy(context.get_admin_context(), volume_id) for volume_id in vols: - yield self.volume.delete_volume(self.context, volume_id) + self.volume.delete_volume(self.context, volume_id) - @defer.inlineCallbacks def test_run_attach_detach_volume(self): """Make sure volume can be attached and detached from instance.""" inst = {} @@ -115,15 +110,15 @@ class VolumeTestCase(test.TrialTestCase): instance_id = db.instance_create(self.context, inst)['id'] mountpoint = "/dev/sdf" volume_id = self._create_volume() - yield self.volume.create_volume(self.context, volume_id) + self.volume.create_volume(self.context, volume_id) if FLAGS.fake_tests: db.volume_attached(self.context, volume_id, instance_id, mountpoint) else: - yield self.compute.attach_volume(self.context, - instance_id, - volume_id, - mountpoint) + self.compute.attach_volume(self.context, + instance_id, + volume_id, + mountpoint) vol = db.volume_get(context.get_admin_context(), volume_id) self.assertEqual(vol['status'], "in-use") self.assertEqual(vol['attach_status'], "attached") @@ -131,25 +126,26 @@ class VolumeTestCase(test.TrialTestCase): instance_ref = db.volume_get_instance(self.context, volume_id) self.assertEqual(instance_ref['id'], instance_id) - self.assertFailure(self.volume.delete_volume(self.context, volume_id), - exception.Error) + self.assertRaises(exception.Error, + self.volume.delete_volume, + self.context, + volume_id) if FLAGS.fake_tests: db.volume_detached(self.context, volume_id) else: - yield self.compute.detach_volume(self.context, - instance_id, - volume_id) + self.compute.detach_volume(self.context, + instance_id, + volume_id) vol = db.volume_get(self.context, volume_id) self.assertEqual(vol['status'], "available") - yield self.volume.delete_volume(self.context, volume_id) + self.volume.delete_volume(self.context, volume_id) self.assertRaises(exception.Error, db.volume_get, self.context, volume_id) db.instance_destroy(self.context, instance_id) - @defer.inlineCallbacks def test_concurrent_volumes_get_different_targets(self): """Ensure multiple concurrent volumes get different targets.""" volume_ids = [] @@ -164,15 +160,11 @@ class VolumeTestCase(test.TrialTestCase): self.assert_(iscsi_target not in targets) targets.append(iscsi_target) logging.debug("Target %s allocated", iscsi_target) - deferreds = [] total_slots = FLAGS.iscsi_num_targets for _index in xrange(total_slots): volume_id = self._create_volume() d = self.volume.create_volume(self.context, volume_id) - d.addCallback(_check) - d.addErrback(self.fail) - deferreds.append(d) - yield defer.DeferredList(deferreds) + _check(d) for volume_id in volume_ids: self.volume.delete_volume(self.context, volume_id) diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py new file mode 100644 index 000000000..ed2e4ffde --- /dev/null +++ b/nova/tests/test_xenapi.py @@ -0,0 +1,220 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Test suite for XenAPI +""" + +import stubout + +from nova import db +from nova import context +from nova import flags +from nova import test +from nova import utils +from nova.auth import manager +from nova.compute import instance_types +from nova.compute import power_state +from nova.virt import xenapi_conn +from nova.virt.xenapi import fake +from nova.virt.xenapi import volume_utils +from nova.tests.db import fakes +from nova.tests.xenapi import stubs + +FLAGS = flags.FLAGS + + +class XenAPIVolumeTestCase(test.TestCase): + """ + Unit tests for Volume operations + """ + def setUp(self): + super(XenAPIVolumeTestCase, self).setUp() + self.stubs = stubout.StubOutForTesting() + FLAGS.target_host = '127.0.0.1' + FLAGS.xenapi_connection_url = 'test_url' + FLAGS.xenapi_connection_password = 'test_pass' + fakes.stub_out_db_instance_api(self.stubs) + stubs.stub_out_get_target(self.stubs) + fake.reset() + self.values = {'name': 1, 'id': 1, + 'project_id': 'fake', + 'user_id': 'fake', + 'image_id': 1, + 'kernel_id': 2, + 'ramdisk_id': 3, + 'instance_type': 'm1.large', + 'mac_address': 'aa:bb:cc:dd:ee:ff', + } + + def _create_volume(self, size='0'): + """Create a volume object.""" + vol = {} + vol['size'] = size + vol['user_id'] = 'fake' + vol['project_id'] = 'fake' + vol['host'] = 'localhost' + vol['availability_zone'] = FLAGS.storage_availability_zone + vol['status'] = "creating" + vol['attach_status'] = "detached" + return db.volume_create(context.get_admin_context(), vol) + + def test_create_iscsi_storage(self): + """ This shows how to test helper classes' methods """ + stubs.stubout_session(self.stubs, stubs.FakeSessionForVolumeTests) + session = xenapi_conn.XenAPISession('test_url', 'root', 'test_pass') + helper = volume_utils.VolumeHelper + helper.XenAPI = session.get_imported_xenapi() + vol = self._create_volume() + info = helper.parse_volume_info(vol['ec2_id'], '/dev/sdc') + label = 'SR-%s' % vol['ec2_id'] + description = 'Test-SR' + sr_ref = helper.create_iscsi_storage(session, info, label, description) + srs = fake.get_all('SR') + self.assertEqual(sr_ref, srs[0]) + db.volume_destroy(context.get_admin_context(), vol['id']) + + def test_parse_volume_info_raise_exception(self): + """ This shows how to test helper classes' methods """ + stubs.stubout_session(self.stubs, stubs.FakeSessionForVolumeTests) + session = xenapi_conn.XenAPISession('test_url', 'root', 'test_pass') + helper = volume_utils.VolumeHelper + helper.XenAPI = session.get_imported_xenapi() + vol = self._create_volume() + # oops, wrong mount point! + self.assertRaises(volume_utils.StorageError, + helper.parse_volume_info, + vol['ec2_id'], + '/dev/sd') + db.volume_destroy(context.get_admin_context(), vol['id']) + + def test_attach_volume(self): + """ This shows how to test Ops classes' methods """ + stubs.stubout_session(self.stubs, stubs.FakeSessionForVolumeTests) + conn = xenapi_conn.get_connection(False) + volume = self._create_volume() + instance = db.instance_create(self.values) + fake.create_vm(instance.name, 'Running') + result = conn.attach_volume(instance.name, volume['ec2_id'], + '/dev/sdc') + + def check(): + # check that the VM has a VBD attached to it + # Get XenAPI reference for the VM + vms = fake.get_all('VM') + # Get XenAPI record for VBD + vbds = fake.get_all('VBD') + vbd = fake.get_record('VBD', vbds[0]) + vm_ref = vbd['VM'] + self.assertEqual(vm_ref, vms[0]) + + check() + + def test_attach_volume_raise_exception(self): + """ This shows how to test when exceptions are raised """ + stubs.stubout_session(self.stubs, + stubs.FakeSessionForVolumeFailedTests) + conn = xenapi_conn.get_connection(False) + volume = self._create_volume() + instance = db.instance_create(self.values) + fake.create_vm(instance.name, 'Running') + self.assertRaises(Exception, + conn.attach_volume, + instance.name, + volume['ec2_id'], + '/dev/sdc') + + def tearDown(self): + super(XenAPIVolumeTestCase, self).tearDown() + self.stubs.UnsetAll() + + +class XenAPIVMTestCase(test.TestCase): + """ + Unit tests for VM operations + """ + def setUp(self): + super(XenAPIVMTestCase, self).setUp() + self.manager = manager.AuthManager() + self.user = self.manager.create_user('fake', 'fake', 'fake', + admin=True) + self.project = self.manager.create_project('fake', 'fake', 'fake') + self.network = utils.import_object(FLAGS.network_manager) + self.stubs = stubout.StubOutForTesting() + FLAGS.xenapi_connection_url = 'test_url' + FLAGS.xenapi_connection_password = 'test_pass' + fake.reset() + fakes.stub_out_db_instance_api(self.stubs) + fake.create_network('fake', FLAGS.flat_network_bridge) + + def test_list_instances_0(self): + stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) + conn = xenapi_conn.get_connection(False) + instances = conn.list_instances() + self.assertEquals(instances, []) + + def test_spawn(self): + stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) + values = {'name': 1, 'id': 1, + 'project_id': self.project.id, + 'user_id': self.user.id, + 'image_id': 1, + 'kernel_id': 2, + 'ramdisk_id': 3, + 'instance_type': 'm1.large', + 'mac_address': 'aa:bb:cc:dd:ee:ff', + } + conn = xenapi_conn.get_connection(False) + instance = db.instance_create(values) + conn.spawn(instance) + + def check(): + instances = conn.list_instances() + self.assertEquals(instances, [1]) + + # Get Nova record for VM + vm_info = conn.get_info(1) + + # Get XenAPI record for VM + vms = fake.get_all('VM') + vm = fake.get_record('VM', vms[0]) + + # Check that m1.large above turned into the right thing. + instance_type = instance_types.INSTANCE_TYPES['m1.large'] + mem_kib = long(instance_type['memory_mb']) << 10 + mem_bytes = str(mem_kib << 10) + vcpus = instance_type['vcpus'] + self.assertEquals(vm_info['max_mem'], mem_kib) + self.assertEquals(vm_info['mem'], mem_kib) + self.assertEquals(vm['memory_static_max'], mem_bytes) + self.assertEquals(vm['memory_dynamic_max'], mem_bytes) + self.assertEquals(vm['memory_dynamic_min'], mem_bytes) + self.assertEquals(vm['VCPUs_max'], str(vcpus)) + self.assertEquals(vm['VCPUs_at_startup'], str(vcpus)) + + # Check that the VM is running according to Nova + self.assertEquals(vm_info['state'], power_state.RUNNING) + + # Check that the VM is running according to XenAPI. + self.assertEquals(vm['power_state'], 'Running') + + check() + + def tearDown(self): + super(XenAPIVMTestCase, self).tearDown() + self.manager.delete_project(self.project) + self.manager.delete_user(self.user) + self.stubs.UnsetAll() diff --git a/nova/tests/xenapi/__init__.py b/nova/tests/xenapi/__init__.py new file mode 100644 index 000000000..1dd02bdc1 --- /dev/null +++ b/nova/tests/xenapi/__init__.py @@ -0,0 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +:mod:`xenapi` -- Stubs for XenAPI +================================= +""" diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py new file mode 100644 index 000000000..a7e592fee --- /dev/null +++ b/nova/tests/xenapi/stubs.py @@ -0,0 +1,103 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Stubouts, mocks and fixtures for the test suite""" + +from nova.virt import xenapi_conn +from nova.virt.xenapi import fake +from nova.virt.xenapi import volume_utils + + +def stubout_session(stubs, cls): + """Stubs out two methods from XenAPISession""" + def fake_import(self): + """Stubs out get_imported_xenapi of XenAPISession""" + fake_module = 'nova.virt.xenapi.fake' + from_list = ['fake'] + return __import__(fake_module, globals(), locals(), from_list, -1) + + stubs.Set(xenapi_conn.XenAPISession, '_create_session', + lambda s, url: cls(url)) + stubs.Set(xenapi_conn.XenAPISession, 'get_imported_xenapi', + fake_import) + + +def stub_out_get_target(stubs): + """Stubs out _get_target in volume_utils""" + def fake_get_target(volume_id): + return (None, None) + + stubs.Set(volume_utils, '_get_target', fake_get_target) + + +class FakeSessionForVMTests(fake.SessionBase): + """ Stubs out a XenAPISession for VM tests """ + def __init__(self, uri): + super(FakeSessionForVMTests, self).__init__(uri) + + def network_get_all_records_where(self, _1, _2): + return self.xenapi.network.get_all_records() + + def host_call_plugin(self, _1, _2, _3, _4, _5): + return '' + + def VM_start(self, _1, ref, _2, _3): + vm = fake.get_record('VM', ref) + if vm['power_state'] != 'Halted': + raise fake.Failure(['VM_BAD_POWER_STATE', ref, 'Halted', + vm['power_state']]) + vm['power_state'] = 'Running' + vm['is_a_template'] = False + vm['is_control_domain'] = False + + +class FakeSessionForVolumeTests(fake.SessionBase): + """ Stubs out a XenAPISession for Volume tests """ + def __init__(self, uri): + super(FakeSessionForVolumeTests, self).__init__(uri) + + def VBD_plug(self, _1, ref): + rec = fake.get_record('VBD', ref) + rec['currently-attached'] = True + + def VDI_introduce(self, _1, uuid, _2, _3, _4, _5, + _6, _7, _8, _9, _10, _11): + valid_vdi = False + refs = fake.get_all('VDI') + for ref in refs: + rec = fake.get_record('VDI', ref) + if rec['uuid'] == uuid: + valid_vdi = True + if not valid_vdi: + raise fake.Failure([['INVALID_VDI', 'session', self._session]]) + + +class FakeSessionForVolumeFailedTests(FakeSessionForVolumeTests): + """ Stubs out a XenAPISession for Volume tests: it injects failures """ + def __init__(self, uri): + super(FakeSessionForVolumeFailedTests, self).__init__(uri) + + def VDI_introduce(self, _1, uuid, _2, _3, _4, _5, + _6, _7, _8, _9, _10, _11): + # This is for testing failure + raise fake.Failure([['INVALID_VDI', 'session', self._session]]) + + def PBD_unplug(self, _1, ref): + rec = fake.get_record('PBD', ref) + rec['currently-attached'] = False + + def SR_forget(self, _1, ref): + pass diff --git a/nova/twistd.py b/nova/twistd.py index cb5648ce6..29be9c4e1 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -43,7 +43,7 @@ else: FLAGS = flags.FLAGS -flags.DEFINE_string('logdir', None, 'directory to keep log files in ' +flags.DEFINE_string('logdir', None, 'directory to keep log files in ' '(will be prepended to $logfile)') @@ -208,7 +208,7 @@ def stop(pidfile): pid = None if not pid: - message = "pidfile %s does not exist. Daemon not running?\n" + message = _("pidfile %s does not exist. Daemon not running?\n") sys.stderr.write(message % pidfile) # Not an error in a restart return @@ -229,7 +229,7 @@ def stop(pidfile): def serve(filename): - logging.debug("Serving %s" % filename) + logging.debug(_("Serving %s") % filename) name = os.path.basename(filename) OptionsClass = WrapTwistedOptions(TwistdServerOptions) options = OptionsClass() @@ -281,7 +281,7 @@ def serve(filename): else: logging.getLogger().setLevel(logging.WARNING) - logging.debug("Full set of FLAGS:") + logging.debug(_("Full set of FLAGS:")) for flag in FLAGS: logging.debug("%s : %s" % (flag, FLAGS.get(flag, None))) diff --git a/nova/utils.py b/nova/utils.py index 9e4ba6bc2..55542f265 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -21,24 +21,24 @@ System-level utilities and helper functions. """ import datetime -import functools import inspect import logging import os import random import subprocess import socket +import struct import sys +import time from xml.sax import saxutils -from twisted.internet.threads import deferToThread +from eventlet import event +from eventlet import greenthread from nova import exception -from nova import flags from nova.exception import ProcessExecutionError -FLAGS = flags.FLAGS TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" @@ -49,7 +49,7 @@ def import_class(import_str): __import__(mod_str) return getattr(sys.modules[mod_str], class_str) except (ImportError, ValueError, AttributeError): - raise exception.NotFound('Class %s cannot be found' % class_str) + raise exception.NotFound(_('Class %s cannot be found') % class_str) def import_object(import_str): @@ -62,8 +62,53 @@ def import_object(import_str): return cls() +def vpn_ping(address, port, timeout=0.05, session_id=None): + """Sends a vpn negotiation packet and returns the server session. + + Returns False on a failure. Basic packet structure is below. + + Client packet (14 bytes):: + 0 1 8 9 13 + +-+--------+-----+ + |x| cli_id |?????| + +-+--------+-----+ + x = packet identifier 0x38 + cli_id = 64 bit identifier + ? = unknown, probably flags/padding + + Server packet (26 bytes):: + 0 1 8 9 13 14 21 2225 + +-+--------+-----+--------+----+ + |x| srv_id |?????| cli_id |????| + +-+--------+-----+--------+----+ + x = packet identifier 0x40 + cli_id = 64 bit identifier + ? = unknown, probably flags/padding + bit 9 was 1 and the rest were 0 in testing + """ + if session_id is None: + session_id = random.randint(0, 0xffffffffffffffff) + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + data = struct.pack("!BQxxxxxx", 0x38, session_id) + sock.sendto(data, (address, port)) + sock.settimeout(timeout) + try: + received = sock.recv(2048) + except socket.timeout: + return False + finally: + sock.close() + fmt = "!BQxxxxxQxxxx" + if len(received) != struct.calcsize(fmt): + print struct.calcsize(fmt) + return False + (identifier, server_sess, client_sess) = struct.unpack(fmt, received) + if identifier == 0x40 and client_sess == session_id: + return server_sess + + def fetchfile(url, target): - logging.debug("Fetching %s" % url) + logging.debug(_("Fetching %s") % url) # c = pycurl.Curl() # fp = open(target, "wb") # c.setopt(c.URL, url) @@ -75,7 +120,7 @@ def fetchfile(url, target): def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): - logging.debug("Running cmd: %s", cmd) + logging.debug(_("Running cmd (subprocess): %s"), cmd) env = os.environ.copy() if addl_env: env.update(addl_env) @@ -88,13 +133,16 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): result = obj.communicate() obj.stdin.close() if obj.returncode: - logging.debug("Result was %s" % (obj.returncode)) if check_exit_code and obj.returncode != 0: (stdout, stderr) = result raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, stderr=stderr, cmd=cmd) + # NOTE(termie): this appears to be necessary to let the subprocess call + # clean something up in between calls, without it two + # execute calls in a row hangs the second one + greenthread.sleep(0) return result @@ -122,14 +170,8 @@ def debug(arg): def runthis(prompt, cmd, check_exit_code=True): - logging.debug("Running %s" % (cmd)) - exit_code = subprocess.call(cmd.split(" ")) - logging.debug(prompt % (exit_code)) - if check_exit_code and exit_code != 0: - raise ProcessExecutionError(exit_code=exit_code, - stdout=None, - stderr=None, - cmd=cmd) + logging.debug(_("Running %s") % (cmd)) + rv, err = execute(cmd, check_exit_code=check_exit_code) def generate_uid(topic, size=8): @@ -159,8 +201,6 @@ def last_octet(address): def get_my_ip(): """Returns the actual ip of the local machine.""" - if getattr(FLAGS, 'fake_tests', None): - return '127.0.0.1' try: csock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) csock.connect(('8.8.8.8', 80)) @@ -168,17 +208,55 @@ def get_my_ip(): csock.close() return addr except socket.gaierror as ex: - logging.warn("Couldn't get IP, using 127.0.0.1 %s", ex) + logging.warn(_("Couldn't get IP, using 127.0.0.1 %s"), ex) return "127.0.0.1" +def utcnow(): + """Overridable version of datetime.datetime.utcnow.""" + if utcnow.override_time: + return utcnow.override_time + return datetime.datetime.utcnow() + + +utcnow.override_time = None + + +def utcnow_ts(): + """Timestamp version of our utcnow function.""" + return time.mktime(utcnow().timetuple()) + + +def set_time_override(override_time=datetime.datetime.utcnow()): + """Override utils.utcnow to return a constant time.""" + utcnow.override_time = override_time + + +def advance_time_delta(timedelta): + """Advance overriden time using a datetime.timedelta.""" + assert(not utcnow.override_time is None) + utcnow.override_time += timedelta + + +def advance_time_seconds(seconds): + """Advance overriden time by seconds.""" + advance_time_delta(datetime.timedelta(0, seconds)) + + +def clear_time_override(): + """Remove the overridden time.""" + utcnow.override_time = None + + def isotime(at=None): + """Returns iso formatted utcnow.""" if not at: - at = datetime.datetime.utcnow() + at = utcnow() return at.strftime(TIME_FORMAT) def parse_isotime(timestr): + """Turn an iso formatted time back into a datetime""" return datetime.datetime.strptime(timestr, TIME_FORMAT) @@ -212,7 +290,7 @@ class LazyPluggable(object): if not self.__backend: backend_name = self.__pivot.value if backend_name not in self.__backends: - raise exception.Error('Invalid backend: %s' % backend_name) + raise exception.Error(_('Invalid backend: %s') % backend_name) backend = self.__backends[backend_name] if type(backend) == type(tuple()): @@ -231,10 +309,41 @@ class LazyPluggable(object): return getattr(backend, key) -def deferredToThread(f): - def g(*args, **kwargs): - return deferToThread(f, *args, **kwargs) - return g +class LoopingCall(object): + def __init__(self, f=None, *args, **kw): + self.args = args + self.kw = kw + self.f = f + self._running = False + + def start(self, interval, now=True): + self._running = True + done = event.Event() + + def _inner(): + if not now: + greenthread.sleep(interval) + try: + while self._running: + self.f(*self.args, **self.kw) + greenthread.sleep(interval) + except Exception: + logging.exception('in looping call') + done.send_exception(*sys.exc_info()) + return + + done.send(True) + + self.done = done + + greenthread.spawn(_inner) + return self.done + + def stop(self): + self._running = False + + def wait(self): + return self.done.wait() def xhtml_escape(value): diff --git a/nova/validate.py b/nova/validate.py deleted file mode 100644 index 7ea27daa6..000000000 --- a/nova/validate.py +++ /dev/null @@ -1,94 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Decorators for argument validation, courtesy of -http://rmi.net/~lutz/rangetest.html""" - - -def rangetest(**argchecks): - """Validate ranges for both + defaults""" - - def onDecorator(func): - """onCall remembers func and argchecks""" - import sys - code = func.__code__ if sys.version_info[0] == 3 else func.func_code - allargs = code.co_varnames[:code.co_argcount] - funcname = func.__name__ - - def onCall(*pargs, **kargs): - # all pargs match first N args by position - # the rest must be in kargs or omitted defaults - positionals = list(allargs) - positionals = positionals[:len(pargs)] - - for (argname, (low, high)) in argchecks.items(): - # for all args to be checked - if argname in kargs: - # was passed by name - if float(kargs[argname]) < low or \ - float(kargs[argname]) > high: - errmsg = '{0} argument "{1}" not in {2}..{3}' - errmsg = errmsg.format(funcname, argname, low, high) - raise TypeError(errmsg) - - elif argname in positionals: - # was passed by position - position = positionals.index(argname) - if float(pargs[position]) < low or \ - float(pargs[position]) > high: - errmsg = '{0} argument "{1}" with value of {4} ' \ - 'not in {2}..{3}' - errmsg = errmsg.format(funcname, argname, low, high, - pargs[position]) - raise TypeError(errmsg) - else: - pass - - return func(*pargs, **kargs) # okay: run original call - return onCall - return onDecorator - - -def typetest(**argchecks): - def onDecorator(func): - import sys - code = func.__code__ if sys.version_info[0] == 3 else func.func_code - allargs = code.co_varnames[:code.co_argcount] - funcname = func.__name__ - - def onCall(*pargs, **kargs): - positionals = list(allargs)[:len(pargs)] - for (argname, typeof) in argchecks.items(): - if argname in kargs: - if not isinstance(kargs[argname], typeof): - errmsg = '{0} argument "{1}" not of type {2}' - errmsg = errmsg.format(funcname, argname, typeof) - raise TypeError(errmsg) - elif argname in positionals: - position = positionals.index(argname) - if not isinstance(pargs[position], typeof): - errmsg = '{0} argument "{1}" with value of {2} ' \ - 'not of type {3}' - errmsg = errmsg.format(funcname, argname, - pargs[position], typeof) - raise TypeError(errmsg) - else: - pass - return func(*pargs, **kargs) - return onCall - return onDecorator diff --git a/nova/virt/connection.py b/nova/virt/connection.py index 11f0fa8ce..61e99944e 100644 --- a/nova/virt/connection.py +++ b/nova/virt/connection.py @@ -25,7 +25,7 @@ import sys from nova import flags from nova.virt import fake from nova.virt import libvirt_conn -from nova.virt import xenapi +from nova.virt import xenapi_conn FLAGS = flags.FLAGS @@ -61,11 +61,11 @@ def get_connection(read_only=False): elif t == 'libvirt': conn = libvirt_conn.get_connection(read_only) elif t == 'xenapi': - conn = xenapi.get_connection(read_only) + conn = xenapi_conn.get_connection(read_only) else: raise Exception('Unknown connection type "%s"' % t) if conn is None: - logging.error('Failed to open connection to the hypervisor') + logging.error(_('Failed to open connection to the hypervisor')) sys.exit(1) return conn diff --git a/nova/virt/fake.py b/nova/virt/fake.py index f855523d3..238acf798 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -25,8 +25,6 @@ semantics of real hypervisor connections. """ -from twisted.internet import defer - from nova import exception from nova.compute import power_state @@ -78,6 +76,12 @@ class FakeConnection(object): cls._instance = cls() return cls._instance + def init_host(self): + """ + Initialize anything that is necessary for the driver to function + """ + return + def list_instances(self): """ Return the names of all the instances known to the virtualization @@ -107,7 +111,6 @@ class FakeConnection(object): fake_instance = FakeInstance() self.instances[instance.name] = fake_instance fake_instance._state = power_state.RUNNING - return defer.succeed(None) def reboot(self, instance): """ @@ -119,19 +122,31 @@ class FakeConnection(object): The work will be done asynchronously. This function returns a Deferred that allows the caller to detect when it is complete. """ - return defer.succeed(None) + pass def rescue(self, instance): """ Rescue the specified instance. """ - return defer.succeed(None) + pass def unrescue(self, instance): """ Unrescue the specified instance. """ - return defer.succeed(None) + pass + + def pause(self, instance, callback): + """ + Pause the specified instance. + """ + pass + + def unpause(self, instance, callback): + """ + Unpause the specified instance. + """ + pass def destroy(self, instance): """ @@ -144,7 +159,6 @@ class FakeConnection(object): Deferred that allows the caller to detect when it is complete. """ del self.instances[instance.name] - return defer.succeed(None) def attach_volume(self, instance_name, device_path, mountpoint): """Attach the disk at device_path to the instance at mountpoint""" @@ -167,7 +181,8 @@ class FakeConnection(object): knowledge of the instance """ if instance_name not in self.instances: - raise exception.NotFound("Instance %s Not Found" % instance_name) + raise exception.NotFound(_("Instance %s Not Found") + % instance_name) i = self.instances[instance_name] return {'state': i._state, 'max_mem': 0, @@ -247,5 +262,6 @@ class FakeConnection(object): class FakeInstance(object): + def __init__(self): self._state = power_state.NOSTATE diff --git a/nova/virt/images.py b/nova/virt/images.py index 981aa5cf3..1c9b2e093 100644 --- a/nova/virt/images.py +++ b/nova/virt/images.py @@ -26,7 +26,7 @@ import time import urlparse from nova import flags -from nova import process +from nova import utils from nova.auth import manager from nova.auth import signer from nova.objectstore import image @@ -50,7 +50,7 @@ def _fetch_s3_image(image, path, user, project): # This should probably move somewhere else, like e.g. a download_as # method on User objects and at the same time get rewritten to use - # twisted web client. + # a web client. headers = {} headers['Date'] = time.strftime("%a, %d %b %Y %H:%M:%S GMT", time.gmtime()) @@ -63,15 +63,16 @@ def _fetch_s3_image(image, path, user, project): cmd = ['/usr/bin/curl', '--fail', '--silent', url] for (k, v) in headers.iteritems(): - cmd += ['-H', '%s: %s' % (k, v)] + cmd += ['-H', '"%s: %s"' % (k, v)] cmd += ['-o', path] - return process.SharedPool().execute(executable=cmd[0], args=cmd[1:]) + cmd_out = ' '.join(cmd) + return utils.execute(cmd_out) def _fetch_local_image(image, path, user, project): source = _image_path('%s/image' % image) - return process.simple_execute('cp %s %s' % (source, path)) + return utils.execute('cp %s %s' % (source, path)) def _image_path(path): diff --git a/nova/virt/libvirt.qemu.xml.template b/nova/virt/libvirt.qemu.xml.template deleted file mode 100644 index 2538b1ade..000000000 --- a/nova/virt/libvirt.qemu.xml.template +++ /dev/null @@ -1,33 +0,0 @@ -<domain type='%(type)s'> - <name>%(name)s</name> - <os> - <type>hvm</type> - <kernel>%(basepath)s/kernel</kernel> - <initrd>%(basepath)s/ramdisk</initrd> - <cmdline>root=/dev/vda1 console=ttyS0</cmdline> - </os> - <features> - <acpi/> - </features> - <memory>%(memory_kb)s</memory> - <vcpu>%(vcpus)s</vcpu> - <devices> - <disk type='file'> - <source file='%(basepath)s/disk'/> - <target dev='vda' bus='virtio'/> - </disk> - <interface type='bridge'> - <source bridge='%(bridge_name)s'/> - <mac address='%(mac_address)s'/> - <!-- <model type='virtio'/> CANT RUN virtio network right now --> - <filterref filter="nova-instance-%(name)s"> - <parameter name="IP" value="%(ip_address)s" /> - <parameter name="DHCPSERVER" value="%(dhcp_server)s" /> - </filterref> - </interface> - <serial type="file"> - <source path='%(basepath)s/console.log'/> - <target port='1'/> - </serial> - </devices> -</domain> diff --git a/nova/virt/libvirt.rescue.qemu.xml.template b/nova/virt/libvirt.rescue.qemu.xml.template deleted file mode 100644 index c0ffbdcee..000000000 --- a/nova/virt/libvirt.rescue.qemu.xml.template +++ /dev/null @@ -1,37 +0,0 @@ -<domain type='%(type)s'> - <name>%(name)s</name> - <os> - <type>hvm</type> - <kernel>%(basepath)s/rescue-kernel</kernel> - <initrd>%(basepath)s/rescue-ramdisk</initrd> - <cmdline>root=/dev/vda1 console=ttyS0</cmdline> - </os> - <features> - <acpi/> - </features> - <memory>%(memory_kb)s</memory> - <vcpu>%(vcpus)s</vcpu> - <devices> - <disk type='file'> - <source file='%(basepath)s/rescue-disk'/> - <target dev='vda' bus='virtio'/> - </disk> - <disk type='file'> - <source file='%(basepath)s/disk'/> - <target dev='vdb' bus='virtio'/> - </disk> - <interface type='bridge'> - <source bridge='%(bridge_name)s'/> - <mac address='%(mac_address)s'/> - <!-- <model type='virtio'/> CANT RUN virtio network right now --> - <filterref filter="nova-instance-%(name)s"> - <parameter name="IP" value="%(ip_address)s" /> - <parameter name="DHCPSERVER" value="%(dhcp_server)s" /> - </filterref> - </interface> - <serial type="file"> - <source path='%(basepath)s/console.log'/> - <target port='1'/> - </serial> - </devices> -</domain> diff --git a/nova/virt/libvirt.rescue.uml.xml.template b/nova/virt/libvirt.rescue.uml.xml.template deleted file mode 100644 index 836f47532..000000000 --- a/nova/virt/libvirt.rescue.uml.xml.template +++ /dev/null @@ -1,26 +0,0 @@ -<domain type='%(type)s'> - <name>%(name)s</name> - <memory>%(memory_kb)s</memory> - <os> - <type>%(type)s</type> - <kernel>/usr/bin/linux</kernel> - <root>/dev/ubda1</root> - </os> - <devices> - <disk type='file'> - <source file='%(basepath)s/rescue-disk'/> - <target dev='ubd0' bus='uml'/> - </disk> - <disk type='file'> - <source file='%(basepath)s/disk'/> - <target dev='ubd1' bus='uml'/> - </disk> - <interface type='bridge'> - <source bridge='%(bridge_name)s'/> - <mac address='%(mac_address)s'/> - </interface> - <console type="file"> - <source path='%(basepath)s/console.log'/> - </console> - </devices> -</domain> diff --git a/nova/virt/libvirt.rescue.xen.xml.template b/nova/virt/libvirt.rescue.xen.xml.template deleted file mode 100644 index 3b8d27237..000000000 --- a/nova/virt/libvirt.rescue.xen.xml.template +++ /dev/null @@ -1,34 +0,0 @@ -<domain type='%(type)s'> - <name>%(name)s</name> - <os> - <type>linux</type> - <kernel>%(basepath)s/kernel</kernel> - <initrd>%(basepath)s/ramdisk</initrd> - <root>/dev/xvda1</root> - <cmdline>ro</cmdline> - </os> - <features> - <acpi/> - </features> - <memory>%(memory_kb)s</memory> - <vcpu>%(vcpus)s</vcpu> - <devices> - <disk type='file'> - <source file='%(basepath)s/rescue-disk'/> - <target dev='sda' /> - </disk> - <disk type='file'> - <source file='%(basepath)s/disk'/> - <target dev='sdb' /> - </disk> - <interface type='bridge'> - <source bridge='%(bridge_name)s'/> - <mac address='%(mac_address)s'/> - </interface> - <console type="file"> - <source path='%(basepath)s/console.log'/> - <target port='1'/> - </console> - </devices> -</domain> - diff --git a/nova/virt/libvirt.uml.xml.template b/nova/virt/libvirt.uml.xml.template deleted file mode 100644 index bb8b47911..000000000 --- a/nova/virt/libvirt.uml.xml.template +++ /dev/null @@ -1,26 +0,0 @@ -<domain type='%(type)s'> - <name>%(name)s</name> - <memory>%(memory_kb)s</memory> - <os> - <type>%(type)s</type> - <kernel>/usr/bin/linux</kernel> - <root>/dev/ubda1</root> - </os> - <devices> - <disk type='file'> - <source file='%(basepath)s/disk'/> - <target dev='ubd0' bus='uml'/> - </disk> - <interface type='bridge'> - <source bridge='%(bridge_name)s'/> - <mac address='%(mac_address)s'/> - <filterref filter="nova-instance-%(name)s"> - <parameter name="IP" value="%(ip_address)s" /> - <parameter name="DHCPSERVER" value="%(dhcp_server)s" /> - </filterref> - </interface> - <console type="file"> - <source path='%(basepath)s/console.log'/> - </console> - </devices> -</domain> diff --git a/nova/virt/libvirt.xen.xml.template b/nova/virt/libvirt.xen.xml.template deleted file mode 100644 index 9677902c6..000000000 --- a/nova/virt/libvirt.xen.xml.template +++ /dev/null @@ -1,30 +0,0 @@ -<domain type='%(type)s'> - <name>%(name)s</name> - <os> - <type>linux</type> - <kernel>%(basepath)s/kernel</kernel> - <initrd>%(basepath)s/ramdisk</initrd> - <root>/dev/xvda1</root> - <cmdline>ro</cmdline> - </os> - <features> - <acpi/> - </features> - <memory>%(memory_kb)s</memory> - <vcpu>%(vcpus)s</vcpu> - <devices> - <disk type='file'> - <source file='%(basepath)s/disk'/> - <target dev='sda' /> - </disk> - <interface type='bridge'> - <source bridge='%(bridge_name)s'/> - <mac address='%(mac_address)s'/> - </interface> - <console type="file"> - <source path='%(basepath)s/console.log'/> - <target port='1'/> - </console> - </devices> -</domain> - diff --git a/nova/virt/libvirt.xml.template b/nova/virt/libvirt.xml.template new file mode 100644 index 000000000..3fb2243da --- /dev/null +++ b/nova/virt/libvirt.xml.template @@ -0,0 +1,79 @@ +<domain type='${type}'> + <name>${name}</name> + <memory>${memory_kb}</memory> + <os> +#if $type == 'uml' + #set $disk_prefix = 'ubd' + #set $disk_bus = 'uml' + <type>uml</type> + <kernel>/usr/bin/linux</kernel> + <root>/dev/ubda1</root> +#else + #if $type == 'xen' + #set $disk_prefix = 'sd' + #set $disk_bus = 'scsi' + <type>linux</type> + <root>/dev/xvda1</root> + #else + #set $disk_prefix = 'vd' + #set $disk_bus = 'virtio' + <type>hvm</type> + #end if + #if $getVar('rescue', False) + <kernel>${basepath}/rescue-kernel</kernel> + <initrd>${basepath}/rescue-ramdisk</initrd> + #else + #if $getVar('kernel', None) + <kernel>${kernel}</kernel> + #if $type == 'xen' + <cmdline>ro</cmdline> + #else + <cmdline>root=/dev/vda1 console=ttyS0</cmdline> + #end if + #if $getVar('ramdisk', None) + <initrd>${ramdisk}</initrd> + #end if + #else + <boot dev="hd" /> + #end if + #end if +#end if + </os> + <features> + <acpi/> + </features> + <vcpu>${vcpus}</vcpu> + <devices> +#if $getVar('rescue', False) + <disk type='file'> + <source file='${basepath}/rescue-disk'/> + <target dev='${disk_prefix}a' bus='${disk_bus}'/> + </disk> + <disk type='file'> + <source file='${basepath}/disk'/> + <target dev='${disk_prefix}b' bus='${disk_bus}'/> + </disk> +#else + <disk type='file'> + <source file='${basepath}/disk'/> + <target dev='${disk_prefix}a' bus='${disk_bus}'/> + </disk> +#end if + <interface type='bridge'> + <source bridge='${bridge_name}'/> + <mac address='${mac_address}'/> + <!-- <model type='virtio'/> CANT RUN virtio network right now --> + <filterref filter="nova-instance-${name}"> + <parameter name="IP" value="${ip_address}" /> + <parameter name="DHCPSERVER" value="${dhcp_server}" /> +#if $getVar('extra_params', False) + ${extra_params} +#end if + </filterref> + </interface> + <serial type="file"> + <source path='${basepath}/console.log'/> + <target port='1'/> + </serial> + </devices> +</domain> diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 6450db8bd..8c2d52bb6 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -27,12 +27,7 @@ Supports KVM, QEMU, UML, and XEN. :libvirt_type: Libvirt domain type. Can be kvm, qemu, uml, xen (default: kvm). :libvirt_uri: Override for the default libvirt URI (depends on libvirt_type). -:libvirt_xml_template: Libvirt XML Template (QEmu/KVM). -:libvirt_xen_xml_template: Libvirt XML Template (Xen). -:libvirt_uml_xml_template: Libvirt XML Template (User Mode Linux). -:libvirt_rescue_xml_template: XML template for rescue mode (KVM & QEMU). -:libvirt_rescue_xen_xml_template: XML templage for rescue mode (XEN). -:libvirt_rescue_uml_xml_template: XML template for rescue mode (UML). +:libvirt_xml_template: Libvirt XML Template. :rescue_image_id: Rescue ami image (default: ami-rescue). :rescue_kernel_id: Rescue aki image (default: aki-rescue). :rescue_ramdisk_id: Rescue ari image (default: ari-rescue). @@ -46,16 +41,16 @@ import os import shutil import re +from eventlet import greenthread +from eventlet import event +from eventlet import tpool + import IPy -from twisted.internet import defer -from twisted.internet import task -from twisted.internet import threads from nova import context from nova import db from nova import exception from nova import flags -from nova import process from nova import utils #from nova.api import context from nova.auth import manager @@ -64,36 +59,20 @@ from nova.compute import instance_types from nova.compute import power_state from nova.virt import images +from Cheetah.Template import Template + libvirt = None libxml2 = None FLAGS = flags.FLAGS -flags.DEFINE_string('libvirt_rescue_xml_template', - utils.abspath('virt/libvirt.rescue.qemu.xml.template'), - 'Libvirt RESCUE XML Template for QEmu/KVM') -flags.DEFINE_string('libvirt_rescue_xen_xml_template', - utils.abspath('virt/libvirt.rescue.xen.xml.template'), - 'Libvirt RESCUE XML Template for xen') -flags.DEFINE_string('libvirt_rescue_uml_xml_template', - utils.abspath('virt/libvirt.rescue.uml.xml.template'), - 'Libvirt RESCUE XML Template for user-mode-linux') # TODO(vish): These flags should probably go into a shared location flags.DEFINE_string('rescue_image_id', 'ami-rescue', 'Rescue ami image') flags.DEFINE_string('rescue_kernel_id', 'aki-rescue', 'Rescue aki image') flags.DEFINE_string('rescue_ramdisk_id', 'ari-rescue', 'Rescue ari image') flags.DEFINE_string('libvirt_xml_template', - utils.abspath('virt/libvirt.qemu.xml.template'), - 'Libvirt XML Template for QEmu/KVM') -flags.DEFINE_string('libvirt_xen_xml_template', - utils.abspath('virt/libvirt.xen.xml.template'), - 'Libvirt XML Template for Xen') -flags.DEFINE_string('libvirt_uml_xml_template', - utils.abspath('virt/libvirt.uml.xml.template'), - 'Libvirt XML Template for user-mode-linux') -flags.DEFINE_string('injected_network_template', - utils.abspath('virt/interfaces.template'), - 'Template file for injected network') + utils.abspath('virt/libvirt.xml.template'), + 'Libvirt XML Template') flags.DEFINE_string('libvirt_type', 'kvm', 'Libvirt domain type (valid options are: ' @@ -122,21 +101,27 @@ def get_connection(read_only): return LibvirtConnection(read_only) +def _get_net_and_mask(cidr): + net = IPy.IP(cidr) + return str(net.net()), str(net.netmask()) + + class LibvirtConnection(object): + def __init__(self, read_only): - (self.libvirt_uri, - template_file, - rescue_file) = self.get_uri_and_templates() + self.libvirt_uri = self.get_uri() - self.libvirt_xml = open(template_file).read() - self.rescue_xml = open(rescue_file).read() + self.libvirt_xml = open(FLAGS.libvirt_xml_template).read() self._wrapped_conn = None self.read_only = read_only + def init_host(self): + NWFilterFirewall(self._conn).setup_base_nwfilters() + @property def _conn(self): if not self._wrapped_conn or not self._test_connection(): - logging.debug('Connecting to libvirt: %s' % self.libvirt_uri) + logging.debug(_('Connecting to libvirt: %s') % self.libvirt_uri) self._wrapped_conn = self._connect(self.libvirt_uri, self.read_only) return self._wrapped_conn @@ -148,24 +133,18 @@ class LibvirtConnection(object): except libvirt.libvirtError as e: if e.get_error_code() == libvirt.VIR_ERR_SYSTEM_ERROR and \ e.get_error_domain() == libvirt.VIR_FROM_REMOTE: - logging.debug('Connection to libvirt broke') + logging.debug(_('Connection to libvirt broke')) return False raise - def get_uri_and_templates(self): + def get_uri(self): if FLAGS.libvirt_type == 'uml': uri = FLAGS.libvirt_uri or 'uml:///system' - template_file = FLAGS.libvirt_uml_xml_template - rescue_file = FLAGS.libvirt_rescue_uml_xml_template elif FLAGS.libvirt_type == 'xen': uri = FLAGS.libvirt_uri or 'xen:///' - template_file = FLAGS.libvirt_xen_xml_template - rescue_file = FLAGS.libvirt_rescue_xen_xml_template else: uri = FLAGS.libvirt_uri or 'qemu:///system' - template_file = FLAGS.libvirt_xml_template - rescue_file = FLAGS.libvirt_rescue_xml_template - return uri, template_file, rescue_file + return uri def _connect(self, uri, read_only): auth = [[libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_NOECHOPROMPT], @@ -188,14 +167,12 @@ class LibvirtConnection(object): except Exception as _err: pass # If the instance is already terminated, we're still happy - d = defer.Deferred() - if cleanup: - d.addCallback(lambda _: self._cleanup(instance)) - # FIXME: What does this comment mean? - # TODO(termie): short-circuit me for tests - # WE'LL save this for when we do shutdown, + + done = event.Event() + + # We'll save this for when we do shutdown, # instead of destroy - but destroy returns immediately - timer = task.LoopingCall(f=None) + timer = utils.LoopingCall(f=None) def _wait_for_shutdown(): try: @@ -204,26 +181,35 @@ class LibvirtConnection(object): instance['id'], state) if state == power_state.SHUTDOWN: timer.stop() - d.callback(None) except Exception: db.instance_set_state(context.get_admin_context(), instance['id'], power_state.SHUTDOWN) timer.stop() - d.callback(None) timer.f = _wait_for_shutdown - timer.start(interval=0.5, now=True) - return d + timer_done = timer.start(interval=0.5, now=True) + + # NOTE(termie): this is strictly superfluous (we could put the + # cleanup code in the timer), but this emulates the + # previous model so I am keeping it around until + # everything has been vetted a bit + def _wait_for_timer(): + timer_done.wait() + if cleanup: + self._cleanup(instance) + done.send() + + greenthread.spawn(_wait_for_timer) + return done def _cleanup(self, instance): target = os.path.join(FLAGS.instances_path, instance['name']) - logging.info('instance %s: deleting instance files %s', + logging.info(_('instance %s: deleting instance files %s'), instance['name'], target) if os.path.exists(target): shutil.rmtree(target) - @defer.inlineCallbacks @exception.wrap_exception def attach_volume(self, instance_name, device_path, mountpoint): virt_dom = self._conn.lookupByName(instance_name) @@ -234,7 +220,6 @@ class LibvirtConnection(object): <target dev='%s' bus='virtio'/> </disk>""" % (device_path, mount_device) virt_dom.attachDevice(xml) - yield def _get_disk_xml(self, xml, device): """Returns the xml for the disk mounted at device""" @@ -256,26 +241,21 @@ class LibvirtConnection(object): if doc != None: doc.freeDoc() - @defer.inlineCallbacks @exception.wrap_exception def detach_volume(self, instance_name, mountpoint): virt_dom = self._conn.lookupByName(instance_name) mount_device = mountpoint.rpartition("/")[2] xml = self._get_disk_xml(virt_dom.XMLDesc(0), mount_device) if not xml: - raise exception.NotFound("No disk at %s" % mount_device) + raise exception.NotFound(_("No disk at %s") % mount_device) virt_dom.detachDevice(xml) - yield - @defer.inlineCallbacks @exception.wrap_exception def reboot(self, instance): - yield self.destroy(instance, False) + self.destroy(instance, False) xml = self.to_xml(instance) - yield self._conn.createXML(xml, 0) - - d = defer.Deferred() - timer = task.LoopingCall(f=None) + self._conn.createXML(xml, 0) + timer = utils.LoopingCall(f=None) def _wait_for_reboot(): try: @@ -283,64 +263,62 @@ class LibvirtConnection(object): db.instance_set_state(context.get_admin_context(), instance['id'], state) if state == power_state.RUNNING: - logging.debug('instance %s: rebooted', instance['name']) + logging.debug(_('instance %s: rebooted'), instance['name']) timer.stop() - d.callback(None) except Exception, exn: - logging.error('_wait_for_reboot failed: %s', exn) + logging.error(_('_wait_for_reboot failed: %s'), exn) db.instance_set_state(context.get_admin_context(), instance['id'], power_state.SHUTDOWN) timer.stop() - d.callback(None) timer.f = _wait_for_reboot - timer.start(interval=0.5, now=True) - yield d + return timer.start(interval=0.5, now=True) + + @exception.wrap_exception + def pause(self, instance, callback): + raise exception.APIError("pause not supported for libvirt.") + + @exception.wrap_exception + def unpause(self, instance, callback): + raise exception.APIError("unpause not supported for libvirt.") - @defer.inlineCallbacks @exception.wrap_exception def rescue(self, instance): - yield self.destroy(instance, False) + self.destroy(instance, False) xml = self.to_xml(instance, rescue=True) rescue_images = {'image_id': FLAGS.rescue_image_id, 'kernel_id': FLAGS.rescue_kernel_id, 'ramdisk_id': FLAGS.rescue_ramdisk_id} - yield self._create_image(instance, xml, 'rescue-', rescue_images) - yield self._conn.createXML(xml, 0) + self._create_image(instance, xml, 'rescue-', rescue_images) + self._conn.createXML(xml, 0) - d = defer.Deferred() - timer = task.LoopingCall(f=None) + timer = utils.LoopingCall(f=None) def _wait_for_rescue(): try: state = self.get_info(instance['name'])['state'] db.instance_set_state(None, instance['id'], state) if state == power_state.RUNNING: - logging.debug('instance %s: rescued', instance['name']) + logging.debug(_('instance %s: rescued'), instance['name']) timer.stop() - d.callback(None) except Exception, exn: - logging.error('_wait_for_rescue failed: %s', exn) + logging.error(_('_wait_for_rescue failed: %s'), exn) db.instance_set_state(None, instance['id'], power_state.SHUTDOWN) timer.stop() - d.callback(None) timer.f = _wait_for_rescue - timer.start(interval=0.5, now=True) - yield d + return timer.start(interval=0.5, now=True) - @defer.inlineCallbacks @exception.wrap_exception def unrescue(self, instance): # NOTE(vish): Because reboot destroys and recreates an instance using # the normal xml file, we can just call reboot here - yield self.reboot(instance) + self.reboot(instance) - @defer.inlineCallbacks @exception.wrap_exception def spawn(self, instance): xml = self.to_xml(instance) @@ -348,14 +326,12 @@ class LibvirtConnection(object): instance['id'], power_state.NOSTATE, 'launching') - yield NWFilterFirewall(self._conn).\ - setup_nwfilters_for_instance(instance) - yield self._create_image(instance, xml) - yield self._conn.createXML(xml, 0) - logging.debug("instance %s: is running", instance['name']) + NWFilterFirewall(self._conn).setup_nwfilters_for_instance(instance) + self._create_image(instance, xml) + self._conn.createXML(xml, 0) + logging.debug(_("instance %s: is running"), instance['name']) - local_d = defer.Deferred() - timer = task.LoopingCall(f=None) + timer = utils.LoopingCall(f=None) def _wait_for_boot(): try: @@ -363,36 +339,33 @@ class LibvirtConnection(object): db.instance_set_state(context.get_admin_context(), instance['id'], state) if state == power_state.RUNNING: - logging.debug('instance %s: booted', instance['name']) + logging.debug(_('instance %s: booted'), instance['name']) timer.stop() - local_d.callback(None) except: - logging.exception('instance %s: failed to boot', + logging.exception(_('instance %s: failed to boot'), instance['name']) db.instance_set_state(context.get_admin_context(), instance['id'], power_state.SHUTDOWN) timer.stop() - local_d.callback(None) + timer.f = _wait_for_boot - timer.start(interval=0.5, now=True) - yield local_d + return timer.start(interval=0.5, now=True) def _flush_xen_console(self, virsh_output): logging.info('virsh said: %r' % (virsh_output,)) virsh_output = virsh_output[0].strip() if virsh_output.startswith('/dev/'): - logging.info('cool, it\'s a device') - d = process.simple_execute("sudo dd if=%s iflag=nonblock" % - virsh_output, check_exit_code=False) - d.addCallback(lambda r: r[0]) - return d + logging.info(_('cool, it\'s a device')) + out, err = utils.execute("sudo dd if=%s iflag=nonblock" % + virsh_output, check_exit_code=False) + return out else: return '' def _append_to_file(self, data, fpath): - logging.info('data: %r, fpath: %r' % (data, fpath)) + logging.info(_('data: %r, fpath: %r') % (data, fpath)) fp = open(fpath, 'a+') fp.write(data) return fpath @@ -407,21 +380,20 @@ class LibvirtConnection(object): def get_console_output(self, instance): console_log = os.path.join(FLAGS.instances_path, instance['name'], 'console.log') - d = process.simple_execute('sudo chown %d %s' % (os.getuid(), - console_log)) + + utils.execute('sudo chown %d %s' % (os.getuid(), console_log)) + if FLAGS.libvirt_type == 'xen': - # Xen is spethial - d.addCallback(lambda _: - process.simple_execute("virsh ttyconsole %s" % - instance['name'])) - d.addCallback(self._flush_xen_console) - d.addCallback(self._append_to_file, console_log) + # Xen is special + virsh_output = utils.execute("virsh ttyconsole %s" % + instance['name']) + data = self._flush_xen_console(virsh_output) + fpath = self._append_to_file(data, console_log) else: - d.addCallback(lambda _: defer.succeed(console_log)) - d.addCallback(self._dump_file) - return d + fpath = console_log + + return self._dump_file(fpath) - @defer.inlineCallbacks def _create_image(self, inst, libvirt_xml, prefix='', disk_images=None): # syntactic nicety basepath = lambda fname = '', prefix = prefix: os.path.join( @@ -430,12 +402,12 @@ class LibvirtConnection(object): prefix + fname) # ensure directories exist and are writable - yield process.simple_execute('mkdir -p %s' % basepath(prefix='')) - yield process.simple_execute('chmod 0777 %s' % basepath(prefix='')) + utils.execute('mkdir -p %s' % basepath(prefix='')) + utils.execute('chmod 0777 %s' % basepath(prefix='')) # TODO(termie): these are blocking calls, it would be great # if they weren't. - logging.info('instance %s: Creating image', inst['name']) + logging.info(_('instance %s: Creating image'), inst['name']) f = open(basepath('libvirt.xml'), 'w') f.write(libvirt_xml) f.close() @@ -452,19 +424,29 @@ class LibvirtConnection(object): 'kernel_id': inst['kernel_id'], 'ramdisk_id': inst['ramdisk_id']} if not os.path.exists(basepath('disk')): - yield images.fetch(inst.image_id, basepath('disk-raw'), user, - project) - if not os.path.exists(basepath('kernel')): - yield images.fetch(inst.kernel_id, basepath('kernel'), user, - project) - if not os.path.exists(basepath('ramdisk')): - yield images.fetch(inst.ramdisk_id, basepath('ramdisk'), user, - project) - - execute = lambda cmd, process_input = None, check_exit_code = True: \ - process.simple_execute(cmd=cmd, - process_input=process_input, - check_exit_code=check_exit_code) + images.fetch(inst.image_id, basepath('disk-raw'), user, + project) + + if inst['kernel_id']: + if not os.path.exists(basepath('kernel')): + images.fetch(inst['kernel_id'], basepath('kernel'), + user, project) + if inst['ramdisk_id']: + if not os.path.exists(basepath('ramdisk')): + images.fetch(inst['ramdisk_id'], basepath('ramdisk'), + user, project) + + def execute(cmd, process_input=None, check_exit_code=True): + return utils.execute(cmd=cmd, + process_input=process_input, + check_exit_code=check_exit_code) + + # For now, we assume that if we're not using a kernel, we're using a + # partitioned disk image where the target partition is the first + # partition + target_partition = None + if not inst['kernel_id']: + target_partition = "1" key = str(inst['key_data']) net = None @@ -481,16 +463,24 @@ class LibvirtConnection(object): 'dns': network_ref['dns']} if key or net: if key: - logging.info('instance %s: injecting key into image %s', + logging.info(_('instance %s: injecting key into image %s'), inst['name'], inst.image_id) if net: - logging.info('instance %s: injecting net into image %s', - inst['name'], inst.image_id) - yield disk.inject_data(basepath('disk-raw'), key, net, - execute=execute) - - if os.path.exists(basepath('disk')): - yield process.simple_execute('rm -f %s' % basepath('disk')) + logging.info(_('instance %s: injecting net into image %s'), + inst['name'], inst.image_id) + try: + disk.inject_data(basepath('disk-raw'), key, net, + partition=target_partition, + execute=execute) + except Exception as e: + # This could be a windows image, or a vmdk format disk + logging.warn(_('instance %s: ignoring error injecting data' + ' into image %s (%s)'), + inst['name'], inst.image_id, e) + + if inst['kernel_id']: + if os.path.exists(basepath('disk')): + utils.execute('rm -f %s' % basepath('disk')) local_bytes = (instance_types.INSTANCE_TYPES[inst.instance_type] ['local_gb'] @@ -499,18 +489,23 @@ class LibvirtConnection(object): resize = True if inst['instance_type'] == 'm1.tiny' or prefix == 'rescue-': resize = False - yield disk.partition(basepath('disk-raw'), basepath('disk'), - local_bytes, resize, execute=execute) + + if inst['kernel_id']: + disk.partition(basepath('disk-raw'), basepath('disk'), + local_bytes, resize, execute=execute) + else: + os.rename(basepath('disk-raw'), basepath('disk')) + disk.extend(basepath('disk'), local_bytes, execute=execute) if FLAGS.libvirt_type == 'uml': - yield process.simple_execute('sudo chown root %s' % - basepath('disk')) + utils.execute('sudo chown root %s' % basepath('disk')) def to_xml(self, instance, rescue=False): # TODO(termie): cache? - logging.debug('instance %s: starting toXML method', instance['name']) - network = db.project_get_network(context.get_admin_context(), - instance['project_id']) + logging.debug(_('instance %s: starting toXML method'), + instance['name']) + network = db.network_get_by_instance(context.get_admin_context(), + instance['id']) # FIXME(vish): stick this in db instance_type = instance['instance_type'] instance_type = instance_types.INSTANCE_TYPES[instance_type] @@ -518,6 +513,15 @@ class LibvirtConnection(object): instance['id']) # Assume that the gateway also acts as the dhcp server. dhcp_server = network['gateway'] + + if FLAGS.allow_project_net_traffic: + net, mask = _get_net_and_mask(network['cidr']) + extra_params = ("<parameter name=\"PROJNET\" value=\"%s\" />\n" + "<parameter name=\"PROJMASK\" value=\"%s\" />\n" + ) % (net, mask) + else: + extra_params = "\n" + xml_info = {'type': FLAGS.libvirt_type, 'name': instance['name'], 'basepath': os.path.join(FLAGS.instances_path, @@ -527,20 +531,30 @@ class LibvirtConnection(object): 'bridge_name': network['bridge'], 'mac_address': instance['mac_address'], 'ip_address': ip_address, - 'dhcp_server': dhcp_server} - if rescue: - libvirt_xml = self.rescue_xml % xml_info - else: - libvirt_xml = self.libvirt_xml % xml_info - logging.debug('instance %s: finished toXML method', instance['name']) + 'dhcp_server': dhcp_server, + 'extra_params': extra_params, + 'rescue': rescue} + if not rescue: + if instance['kernel_id']: + xml_info['kernel'] = xml_info['basepath'] + "/kernel" + + if instance['ramdisk_id']: + xml_info['ramdisk'] = xml_info['basepath'] + "/ramdisk" + + xml_info['disk'] = xml_info['basepath'] + "/disk" - return libvirt_xml + xml = str(Template(self.libvirt_xml, searchList=[xml_info])) + logging.debug(_('instance %s: finished toXML method'), + instance['name']) + + return xml def get_info(self, instance_name): try: virt_dom = self._conn.lookupByName(instance_name) except: - raise exception.NotFound("Instance %s not found" % instance_name) + raise exception.NotFound(_("Instance %s not found") + % instance_name) (state, max_mem, mem, num_cpu, cpu_time) = virt_dom.info() return {'state': state, 'max_mem': max_mem, @@ -644,19 +658,18 @@ class LibvirtConnection(object): """ Get hypervisor version """ return self._conn.getVersion() - def get_cpu_xml(self): + def get_cpu_xml(self): """ Get cpuinfo information """ xmlstr = self._conn.getCapabilities() xml = libxml2.parseDoc(xmlstr) nodes = xml.xpathEval('//cpu') if 1 != len(nodes): msg = 'Unexpected xml format. tag "cpu" must be 1, but %d.' % len(nodes) - msg += '\n'+xml.serialize() - raise exception.Invalid(msg) + msg += '\n' + xml.serialize() + raise exception.Invalid(_(msg)) cpuxmlstr = re.sub("\n|[ ]+", ' ', nodes[0].serialize()) return cpuxmlstr - def block_stats(self, instance_name, disk): """ Note that this function takes an instance name, not an Instance, so @@ -673,17 +686,15 @@ class LibvirtConnection(object): domain = self._conn.lookupByName(instance_name) return domain.interfaceStats(interface) - def refresh_security_group(self, security_group_id): fw = NWFilterFirewall(self._conn) fw.ensure_security_group_filter(security_group_id) - def setup_nwfilters_for_instance(self, instance): + """ See same method of NWFilterFirewall class """ nwfilter = NWFilterFirewall(self._conn) return nwfilter.setup_nwfilters_for_instance(instance) - def nwfilter_for_instance_exists(self, instance_ref): try: filter = 'nova-instance-%s' % instance_ref.name @@ -692,7 +703,6 @@ class LibvirtConnection(object): except libvirt.libvirtError: return False - def compareCPU(self, xml): """ Check the host cpu is compatible to a cpu given by xml. @@ -702,86 +712,95 @@ class LibvirtConnection(object): 'http://libvirt.org/html/libvirt-libvirt.html#virCPUCompareResult' """ - return self._conn.compareCPU(xml,0) + return self._conn.compareCPU(xml, 0) + def live_migration(self, context, instance_ref, dest): + """ + Just spawning live_migration operation for + distributing high-load. + """ + greenthread.spawn(self._live_migration, context, instance_ref, dest) + + def _live_migration(self, context, instance_ref, dest): + """ Do live migration.""" + + # Do live migration. + try: + uri = FLAGS.live_migration_uri % dest + out, err = utils.execute("sudo virsh migrate --live %s %s" + % (instance_ref.name, uri)) + except exception.ProcessExecutionError: + id = instance_ref['id'] + db.instance_set_state(context, id, power_state.RUNNING, 'running') + raise - def live_migration(self, instance_ref, dest): - uri = FLAGS.live_migration_uri % dest - out, err = utils.execute("sudo virsh migrate --live %s %s" - % (instance_ref.name, uri)) + # Waiting for completion of live_migration. + timer = utils.LoopingCall(f=None) - # wait for completion of live_migration - d = defer.Deferred() - d.addCallback(lambda _: self._post_live_migration(instance_ref, dest)) - timer = task.LoopingCall(f=None) + def wait_for_live_migration(): - def _wait_for_live_migration(): try: state = self.get_info(instance_ref.name)['state'] - #except libvirt.libvirtError, e: except exception.NotFound: timer.stop() - d.callback(None) - timer.f = _wait_for_live_migration - timer.start(interval=0.5, now=True) - return d + self._post_live_migration(context, instance_ref, dest) - def _post_live_migration(self, instance_ref, dest): + timer.f = wait_for_live_migration + timer.start(interval=0.5, now=True) - # 1. detaching volumes + def _post_live_migration(self, context, instance_ref, dest): + """ + Post operations for live migration. + Mainly, database updating. + """ + # Detaching volumes. # (not necessary in current version ) - #try : - # ec2_id = instance_ref['ec2_id'] - # volumes = db.volume_get_by_ec2_id(context, ec2_id) - # for volume in volumes : - # self.detach_volume(context, instance_id, volume.id) - #except exception.NotFound: - # logging.debug('%s doesnt mount any volumes.. ' % ec2_id) - - # 2. releasing vlan + + # Releasing vlan. # (not necessary in current implementation?) - # 3. releasing security group ingress rule + # Releasing security group ingress rule. # (not necessary in current implementation?) - # 4. database updating + # Database updating. ec2_id = instance_ref['hostname'] - ctxt = context.get_admin_context() instance_id = instance_ref['id'] - fixed_ip = db.instance_get_fixed_address(ctxt, instance_id) - # not return if fixed_ip is not found, otherwise, + fixed_ip = db.instance_get_fixed_address(context, instance_id) + # Not return if fixed_ip is not found, otherwise, # instance never be accessible.. if None == fixed_ip: - logging.error('fixed_ip is not found for %s ' % ec2_id) - db.fixed_ip_update(ctxt, fixed_ip, {'host': dest}) - network_ref = db.fixed_ip_get_network(ctxt, fixed_ip) - db.network_update(ctxt, network_ref['id'], {'host': dest}) + logging.warn('fixed_ip is not found for %s ' % ec2_id) + db.fixed_ip_update(context, fixed_ip, {'host': dest}) + network_ref = db.fixed_ip_get_network(context, fixed_ip) + db.network_update(context, network_ref['id'], {'host': dest}) try: - floating_ip = db.instance_get_floating_address(ctxt, instance_id) - # not return if floating_ip is not found, otherwise, + floating_ip = db.instance_get_floating_address(context, instance_id) + # Not return if floating_ip is not found, otherwise, # instance never be accessible.. if None == floating_ip: logging.error('floating_ip is not found for %s ' % ec2_id) - floating_ip_ref = db.floating_ip_get_by_address(ctxt, floating_ip) - db.floating_ip_update(ctxt, - floating_ip_ref['address'], - {'host': dest}) + else: + floating_ip_ref = db.floating_ip_get_by_address(context, + floating_ip) + db.floating_ip_update(context, + floating_ip_ref['address'], + {'host': dest}) except exception.NotFound: logging.debug('%s doesnt have floating_ip.. ' % ec2_id) except: msg = 'Live migration: Unexpected error:' msg += '%s cannot inherit floating ip.. ' % ec2_id - logging.error(msg) + logging.error(_(msg)) - db.instance_update(ctxt, + db.instance_update(context, instance_id, {'state_description': 'running', 'state': power_state.RUNNING, 'host': dest}) - logging.info('Live migrating %s to %s finishes successfully' + logging.info(_('Live migrating %s to %s finishes successfully') % (ec2_id, dest)) @@ -858,6 +877,14 @@ class NWFilterFirewall(object): </rule> </filter>''' + nova_vpn_filter = '''<filter name='nova-vpn' chain='root'> + <uuid>2086015e-cf03-11df-8c5d-080027c27973</uuid> + <filterref filter='allow-dhcp-server'/> + <filterref filter='nova-allow-dhcp-server'/> + <filterref filter='nova-base-ipv4'/> + <filterref filter='nova-base-ipv6'/> + </filter>''' + def nova_base_ipv4_filter(self): retval = "<filter name='nova-base-ipv4' chain='ipv4'>" for protocol in ['tcp', 'udp', 'icmp']: @@ -882,27 +909,31 @@ class NWFilterFirewall(object): retval += '</filter>' return retval - def nova_project_filter(self, project, net, mask): - retval = "<filter name='nova-project-%s' chain='ipv4'>" % project + def nova_project_filter(self): + retval = "<filter name='nova-project' chain='ipv4'>" for protocol in ['tcp', 'udp', 'icmp']: retval += """<rule action='accept' direction='in' priority='200'> - <%s srcipaddr='%s' srcipmask='%s' /> - </rule>""" % (protocol, net, mask) + <%s srcipaddr='$PROJNET' srcipmask='$PROJMASK' /> + </rule>""" % protocol retval += '</filter>' return retval def _define_filter(self, xml): if callable(xml): xml = xml() - d = threads.deferToThread(self._conn.nwfilterDefineXML, xml) - return d - @staticmethod - def _get_net_and_mask(cidr): - net = IPy.IP(cidr) - return str(net.net()), str(net.netmask()) + # execute in a native thread and block current greenthread until done + tpool.execute(self._conn.nwfilterDefineXML, xml) + + def setup_base_nwfilters(self): + self._define_filter(self.nova_base_ipv4_filter) + self._define_filter(self.nova_base_ipv6_filter) + self._define_filter(self.nova_dhcp_filter) + self._define_filter(self.nova_base_filter) + self._define_filter(self.nova_vpn_filter) + if FLAGS.allow_project_net_traffic: + self._define_filter(self.nova_project_filter) - @defer.inlineCallbacks def setup_nwfilters_for_instance(self, instance): """ Creates an NWFilter for the given instance. In the process, @@ -910,35 +941,25 @@ class NWFilterFirewall(object): the base filter are all in place. """ - yield self._define_filter(self.nova_base_ipv4_filter) - yield self._define_filter(self.nova_base_ipv6_filter) - yield self._define_filter(self.nova_dhcp_filter) - yield self._define_filter(self.nova_base_filter) + nwfilter_xml = ("<filter name='nova-instance-%s' chain='root'>\n" + ) % instance['name'] - nwfilter_xml = "<filter name='nova-instance-%s' chain='root'>\n" \ - " <filterref filter='nova-base' />\n" % \ - instance['name'] + if instance['image_id'] == FLAGS.vpn_image_id: + nwfilter_xml += " <filterref filter='nova-vpn' />\n" + else: + nwfilter_xml += " <filterref filter='nova-base' />\n" if FLAGS.allow_project_net_traffic: - network_ref = db.project_get_network(context.get_admin_context(), - instance['project_id']) - net, mask = self._get_net_and_mask(network_ref['cidr']) - project_filter = self.nova_project_filter(instance['project_id'], - net, mask) - yield self._define_filter(project_filter) - - nwfilter_xml += " <filterref filter='nova-project-%s' />\n" % \ - instance['project_id'] + nwfilter_xml += " <filterref filter='nova-project' />\n" for security_group in instance.security_groups: - yield self.ensure_security_group_filter(security_group['id']) + self.ensure_security_group_filter(security_group['id']) - nwfilter_xml += " <filterref filter='nova-secgroup-%d' />\n" % \ - security_group['id'] + nwfilter_xml += (" <filterref filter='nova-secgroup-%d' />\n" + ) % security_group['id'] nwfilter_xml += "</filter>" - yield self._define_filter(nwfilter_xml) - return + self._define_filter(nwfilter_xml) def ensure_security_group_filter(self, security_group_id): return self._define_filter( @@ -951,7 +972,7 @@ class NWFilterFirewall(object): for rule in security_group.rules: rule_xml += "<rule action='accept' direction='in' priority='300'>" if rule.cidr: - net, mask = self._get_net_and_mask(rule.cidr) + net, mask = _get_net_and_mask(rule.cidr) rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \ (rule.protocol, net, mask) if rule.protocol in ['tcp', 'udp']: diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py deleted file mode 100644 index de3d68582..000000000 --- a/nova/virt/xenapi.py +++ /dev/null @@ -1,442 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright (c) 2010 Citrix Systems, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -A connection to XenServer or Xen Cloud Platform. - -The concurrency model for this class is as follows: - -All XenAPI calls are on a thread (using t.i.t.deferToThread, via the decorator -deferredToThread). They are remote calls, and so may hang for the usual -reasons. They should not be allowed to block the reactor thread. - -All long-running XenAPI calls (VM.start, VM.reboot, etc) are called async -(using XenAPI.VM.async_start etc). These return a task, which can then be -polled for completion. Polling is handled using reactor.callLater. - -This combination of techniques means that we don't block the reactor thread at -all, and at the same time we don't hold lots of threads waiting for -long-running operations. - -FIXME: get_info currently doesn't conform to these rules, and will block the -reactor thread if the VM.get_by_name_label or VM.get_record calls block. - -**Related Flags** - -:xenapi_connection_url: URL for connection to XenServer/Xen Cloud Platform. -:xenapi_connection_username: Username for connection to XenServer/Xen Cloud - Platform (default: root). -:xenapi_connection_password: Password for connection to XenServer/Xen Cloud - Platform. -:xenapi_task_poll_interval: The interval (seconds) used for polling of - remote tasks (Async.VM.start, etc) - (default: 0.5). - -""" - -import logging -import xmlrpclib - -from twisted.internet import defer -from twisted.internet import reactor - -from nova import db -from nova import flags -from nova import utils -from nova.auth.manager import AuthManager -from nova.compute import instance_types -from nova.compute import power_state -from nova.virt import images - -XenAPI = None - - -FLAGS = flags.FLAGS -flags.DEFINE_string('xenapi_connection_url', - None, - 'URL for connection to XenServer/Xen Cloud Platform.' - ' Required if connection_type=xenapi.') -flags.DEFINE_string('xenapi_connection_username', - 'root', - 'Username for connection to XenServer/Xen Cloud Platform.' - ' Used only if connection_type=xenapi.') -flags.DEFINE_string('xenapi_connection_password', - None, - 'Password for connection to XenServer/Xen Cloud Platform.' - ' Used only if connection_type=xenapi.') -flags.DEFINE_float('xenapi_task_poll_interval', - 0.5, - 'The interval used for polling of remote tasks ' - '(Async.VM.start, etc). Used only if ' - 'connection_type=xenapi.') - - -XENAPI_POWER_STATE = { - 'Halted': power_state.SHUTDOWN, - 'Running': power_state.RUNNING, - 'Paused': power_state.PAUSED, - 'Suspended': power_state.SHUTDOWN, # FIXME - 'Crashed': power_state.CRASHED} - - -def get_connection(_): - """Note that XenAPI doesn't have a read-only connection mode, so - the read_only parameter is ignored.""" - # This is loaded late so that there's no need to install this - # library when not using XenAPI. - global XenAPI - if XenAPI is None: - XenAPI = __import__('XenAPI') - url = FLAGS.xenapi_connection_url - username = FLAGS.xenapi_connection_username - password = FLAGS.xenapi_connection_password - if not url or password is None: - raise Exception('Must specify xenapi_connection_url, ' - 'xenapi_connection_username (optionally), and ' - 'xenapi_connection_password to use ' - 'connection_type=xenapi') - return XenAPIConnection(url, username, password) - - -class XenAPIConnection(object): - def __init__(self, url, user, pw): - self._conn = XenAPI.Session(url) - self._conn.login_with_password(user, pw) - - def list_instances(self): - return [self._conn.xenapi.VM.get_name_label(vm) \ - for vm in self._conn.xenapi.VM.get_all()] - - @defer.inlineCallbacks - def spawn(self, instance): - vm = yield self._lookup(instance.name) - if vm is not None: - raise Exception('Attempted to create non-unique name %s' % - instance.name) - - network = db.project_get_network(None, instance.project_id) - network_ref = \ - yield self._find_network_with_bridge(network.bridge) - - user = AuthManager().get_user(instance.user_id) - project = AuthManager().get_project(instance.project_id) - vdi_uuid = yield self._fetch_image( - instance.image_id, user, project, True) - kernel = yield self._fetch_image( - instance.kernel_id, user, project, False) - ramdisk = yield self._fetch_image( - instance.ramdisk_id, user, project, False) - vdi_ref = yield self._call_xenapi('VDI.get_by_uuid', vdi_uuid) - - vm_ref = yield self._create_vm(instance, kernel, ramdisk) - yield self._create_vbd(vm_ref, vdi_ref, 0, True) - if network_ref: - yield self._create_vif(vm_ref, network_ref, instance.mac_address) - logging.debug('Starting VM %s...', vm_ref) - yield self._call_xenapi('VM.start', vm_ref, False, False) - logging.info('Spawning VM %s created %s.', instance.name, vm_ref) - - @defer.inlineCallbacks - def _create_vm(self, instance, kernel, ramdisk): - """Create a VM record. Returns a Deferred that gives the new - VM reference.""" - - instance_type = instance_types.INSTANCE_TYPES[instance.instance_type] - mem = str(long(instance_type['memory_mb']) * 1024 * 1024) - vcpus = str(instance_type['vcpus']) - rec = { - 'name_label': instance.name, - 'name_description': '', - 'is_a_template': False, - 'memory_static_min': '0', - 'memory_static_max': mem, - 'memory_dynamic_min': mem, - 'memory_dynamic_max': mem, - 'VCPUs_at_startup': vcpus, - 'VCPUs_max': vcpus, - 'VCPUs_params': {}, - 'actions_after_shutdown': 'destroy', - 'actions_after_reboot': 'restart', - 'actions_after_crash': 'destroy', - 'PV_bootloader': '', - 'PV_kernel': kernel, - 'PV_ramdisk': ramdisk, - 'PV_args': 'root=/dev/xvda1', - 'PV_bootloader_args': '', - 'PV_legacy_args': '', - 'HVM_boot_policy': '', - 'HVM_boot_params': {}, - 'platform': {}, - 'PCI_bus': '', - 'recommendations': '', - 'affinity': '', - 'user_version': '0', - 'other_config': {}, - } - logging.debug('Created VM %s...', instance.name) - vm_ref = yield self._call_xenapi('VM.create', rec) - logging.debug('Created VM %s as %s.', instance.name, vm_ref) - defer.returnValue(vm_ref) - - @defer.inlineCallbacks - def _create_vbd(self, vm_ref, vdi_ref, userdevice, bootable): - """Create a VBD record. Returns a Deferred that gives the new - VBD reference.""" - - vbd_rec = {} - vbd_rec['VM'] = vm_ref - vbd_rec['VDI'] = vdi_ref - vbd_rec['userdevice'] = str(userdevice) - vbd_rec['bootable'] = bootable - vbd_rec['mode'] = 'RW' - vbd_rec['type'] = 'disk' - vbd_rec['unpluggable'] = True - vbd_rec['empty'] = False - vbd_rec['other_config'] = {} - vbd_rec['qos_algorithm_type'] = '' - vbd_rec['qos_algorithm_params'] = {} - vbd_rec['qos_supported_algorithms'] = [] - logging.debug('Creating VBD for VM %s, VDI %s ... ', vm_ref, vdi_ref) - vbd_ref = yield self._call_xenapi('VBD.create', vbd_rec) - logging.debug('Created VBD %s for VM %s, VDI %s.', vbd_ref, vm_ref, - vdi_ref) - defer.returnValue(vbd_ref) - - @defer.inlineCallbacks - def _create_vif(self, vm_ref, network_ref, mac_address): - """Create a VIF record. Returns a Deferred that gives the new - VIF reference.""" - - vif_rec = {} - vif_rec['device'] = '0' - vif_rec['network'] = network_ref - vif_rec['VM'] = vm_ref - vif_rec['MAC'] = mac_address - vif_rec['MTU'] = '1500' - vif_rec['other_config'] = {} - vif_rec['qos_algorithm_type'] = '' - vif_rec['qos_algorithm_params'] = {} - logging.debug('Creating VIF for VM %s, network %s ... ', vm_ref, - network_ref) - vif_ref = yield self._call_xenapi('VIF.create', vif_rec) - logging.debug('Created VIF %s for VM %s, network %s.', vif_ref, - vm_ref, network_ref) - defer.returnValue(vif_ref) - - @defer.inlineCallbacks - def _find_network_with_bridge(self, bridge): - expr = 'field "bridge" = "%s"' % bridge - networks = yield self._call_xenapi('network.get_all_records_where', - expr) - if len(networks) == 1: - defer.returnValue(networks.keys()[0]) - elif len(networks) > 1: - raise Exception('Found non-unique network for bridge %s' % bridge) - else: - raise Exception('Found no network for bridge %s' % bridge) - - @defer.inlineCallbacks - def _fetch_image(self, image, user, project, use_sr): - """use_sr: True to put the image as a VDI in an SR, False to place - it on dom0's filesystem. The former is for VM disks, the latter for - its kernel and ramdisk (if external kernels are being used). - Returns a Deferred that gives the new VDI UUID.""" - - url = images.image_url(image) - access = AuthManager().get_access_key(user, project) - logging.debug("Asking xapi to fetch %s as %s" % (url, access)) - fn = use_sr and 'get_vdi' or 'get_kernel' - args = {} - args['src_url'] = url - args['username'] = access - args['password'] = user.secret - if use_sr: - args['add_partition'] = 'true' - task = yield self._async_call_plugin('objectstore', fn, args) - uuid = yield self._wait_for_task(task) - defer.returnValue(uuid) - - @defer.inlineCallbacks - def reboot(self, instance): - vm = yield self._lookup(instance.name) - if vm is None: - raise Exception('instance not present %s' % instance.name) - task = yield self._call_xenapi('Async.VM.clean_reboot', vm) - yield self._wait_for_task(task) - - @defer.inlineCallbacks - def destroy(self, instance): - vm = yield self._lookup(instance.name) - if vm is None: - # Don't complain, just return. This lets us clean up instances - # that have already disappeared from the underlying platform. - defer.returnValue(None) - # Get the VDIs related to the VM - vdis = yield self._lookup_vm_vdis(vm) - try: - task = yield self._call_xenapi('Async.VM.hard_shutdown', vm) - yield self._wait_for_task(task) - except Exception, exc: - logging.warn(exc) - # Disk clean-up - if vdis: - for vdi in vdis: - try: - task = yield self._call_xenapi('Async.VDI.destroy', vdi) - yield self._wait_for_task(task) - except Exception, exc: - logging.warn(exc) - try: - task = yield self._call_xenapi('Async.VM.destroy', vm) - yield self._wait_for_task(task) - except Exception, exc: - logging.warn(exc) - - def get_info(self, instance_id): - vm = self._lookup_blocking(instance_id) - if vm is None: - raise Exception('instance not present %s' % instance_id) - rec = self._conn.xenapi.VM.get_record(vm) - return {'state': XENAPI_POWER_STATE[rec['power_state']], - 'max_mem': long(rec['memory_static_max']) >> 10, - 'mem': long(rec['memory_dynamic_max']) >> 10, - 'num_cpu': rec['VCPUs_max'], - 'cpu_time': 0} - - def get_console_output(self, instance): - return 'FAKE CONSOLE OUTPUT' - - @utils.deferredToThread - def _lookup(self, i): - return self._lookup_blocking(i) - - def _lookup_blocking(self, i): - vms = self._conn.xenapi.VM.get_by_name_label(i) - n = len(vms) - if n == 0: - return None - elif n > 1: - raise Exception('duplicate name found: %s' % i) - else: - return vms[0] - - @utils.deferredToThread - def _lookup_vm_vdis(self, vm): - return self._lookup_vm_vdis_blocking(vm) - - def _lookup_vm_vdis_blocking(self, vm): - # Firstly we get the VBDs, then the VDIs. - # TODO: do we leave the read-only devices? - vbds = self._conn.xenapi.VM.get_VBDs(vm) - vdis = [] - if vbds: - for vbd in vbds: - try: - vdi = self._conn.xenapi.VBD.get_VDI(vbd) - # Test valid VDI - record = self._conn.xenapi.VDI.get_record(vdi) - except Exception, exc: - logging.warn(exc) - else: - vdis.append(vdi) - if len(vdis) > 0: - return vdis - else: - return None - - def _wait_for_task(self, task): - """Return a Deferred that will give the result of the given task. - The task is polled until it completes.""" - d = defer.Deferred() - reactor.callLater(0, self._poll_task, task, d) - return d - - @utils.deferredToThread - def _poll_task(self, task, deferred): - """Poll the given XenAPI task, and fire the given Deferred if we - get a result.""" - try: - #logging.debug('Polling task %s...', task) - status = self._conn.xenapi.task.get_status(task) - if status == 'pending': - reactor.callLater(FLAGS.xenapi_task_poll_interval, - self._poll_task, task, deferred) - elif status == 'success': - result = self._conn.xenapi.task.get_result(task) - logging.info('Task %s status: success. %s', task, result) - deferred.callback(_parse_xmlrpc_value(result)) - else: - error_info = self._conn.xenapi.task.get_error_info(task) - logging.warn('Task %s status: %s. %s', task, status, - error_info) - deferred.errback(XenAPI.Failure(error_info)) - #logging.debug('Polling task %s done.', task) - except Exception, exc: - logging.warn(exc) - deferred.errback(exc) - - @utils.deferredToThread - def _call_xenapi(self, method, *args): - """Call the specified XenAPI method on a background thread. Returns - a Deferred for the result.""" - f = self._conn.xenapi - for m in method.split('.'): - f = f.__getattr__(m) - return f(*args) - - @utils.deferredToThread - def _async_call_plugin(self, plugin, fn, args): - """Call Async.host.call_plugin on a background thread. Returns a - Deferred with the task reference.""" - return _unwrap_plugin_exceptions( - self._conn.xenapi.Async.host.call_plugin, - self._get_xenapi_host(), plugin, fn, args) - - def _get_xenapi_host(self): - return self._conn.xenapi.session.get_this_host(self._conn.handle) - - -def _unwrap_plugin_exceptions(func, *args, **kwargs): - try: - return func(*args, **kwargs) - except XenAPI.Failure, exc: - logging.debug("Got exception: %s", exc) - if (len(exc.details) == 4 and - exc.details[0] == 'XENAPI_PLUGIN_EXCEPTION' and - exc.details[2] == 'Failure'): - params = None - try: - params = eval(exc.details[3]) - except: - raise exc - raise XenAPI.Failure(params) - else: - raise - except xmlrpclib.ProtocolError, exc: - logging.debug("Got exception: %s", exc) - raise - - -def _parse_xmlrpc_value(val): - """Parse the given value as if it were an XML-RPC value. This is - sometimes used as the format for the task.result field.""" - if not val: - return val - x = xmlrpclib.loads( - '<?xml version="1.0"?><methodResponse><params><param>' + - val + - '</param></params></methodResponse>') - return x[0][0] diff --git a/nova/virt/xenapi/__init__.py b/nova/virt/xenapi/__init__.py new file mode 100644 index 000000000..c75162f08 --- /dev/null +++ b/nova/virt/xenapi/__init__.py @@ -0,0 +1,30 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +:mod:`xenapi` -- Nova support for XenServer and XCP through XenAPI +================================================================== +""" + + +class HelperBase(object): + """ + The base for helper classes. This adds the XenAPI class attribute + """ + XenAPI = None + + def __init__(self): + return diff --git a/nova/virt/xenapi/fake.py b/nova/virt/xenapi/fake.py new file mode 100644 index 000000000..7a6c9ee71 --- /dev/null +++ b/nova/virt/xenapi/fake.py @@ -0,0 +1,388 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 +# +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +#============================================================================ +# +# Parts of this file are based upon xmlrpclib.py, the XML-RPC client +# interface included in the Python distribution. +# +# Copyright (c) 1999-2002 by Secret Labs AB +# Copyright (c) 1999-2002 by Fredrik Lundh +# +# By obtaining, using, and/or copying this software and/or its +# associated documentation, you agree that you have read, understood, +# and will comply with the following terms and conditions: +# +# Permission to use, copy, modify, and distribute this software and +# its associated documentation for any purpose and without fee is +# hereby granted, provided that the above copyright notice appears in +# all copies, and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of +# Secret Labs AB or the author not be used in advertising or publicity +# pertaining to distribution of the software without specific, written +# prior permission. +# +# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD +# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- +# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR +# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY +# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# -------------------------------------------------------------------- + + +""" +A fake XenAPI SDK. +""" + + +import datetime +import logging +import uuid + +from nova import exception + + +_CLASSES = ['host', 'network', 'session', 'SR', 'VBD',\ + 'PBD', 'VDI', 'VIF', 'VM', 'task'] + +_db_content = {} + + +def reset(): + for c in _CLASSES: + _db_content[c] = {} + create_host('fake') + + +def create_host(name_label): + return _create_object('host', { + 'name_label': name_label, + }) + + +def create_network(name_label, bridge): + return _create_object('network', { + 'name_label': name_label, + 'bridge': bridge, + }) + + +def create_vm(name_label, status, + is_a_template=False, is_control_domain=False): + return _create_object('VM', { + 'name_label': name_label, + 'power-state': status, + 'is_a_template': is_a_template, + 'is_control_domain': is_control_domain, + }) + + +def create_vdi(name_label, read_only, sr_ref, sharable): + return _create_object('VDI', { + 'name_label': name_label, + 'read_only': read_only, + 'SR': sr_ref, + 'type': '', + 'name_description': '', + 'sharable': sharable, + 'other_config': {}, + 'location': '', + 'xenstore_data': '', + 'sm_config': {}, + 'VBDs': {}, + }) + + +def create_pbd(config, sr_ref, attached): + return _create_object('PBD', { + 'device-config': config, + 'SR': sr_ref, + 'currently-attached': attached, + }) + + +def create_task(name_label): + return _create_object('task', { + 'name_label': name_label, + 'status': 'pending', + }) + + +def _create_object(table, obj): + ref = str(uuid.uuid4()) + obj['uuid'] = str(uuid.uuid4()) + _db_content[table][ref] = obj + return ref + + +def _create_sr(table, obj): + sr_type = obj[6] + # Forces fake to support iscsi only + if sr_type != 'iscsi': + raise Failure(['SR_UNKNOWN_DRIVER', sr_type]) + sr_ref = _create_object(table, obj[2]) + vdi_ref = create_vdi('', False, sr_ref, False) + pbd_ref = create_pbd('', sr_ref, True) + _db_content['SR'][sr_ref]['VDIs'] = [vdi_ref] + _db_content['SR'][sr_ref]['PBDs'] = [pbd_ref] + _db_content['VDI'][vdi_ref]['SR'] = sr_ref + _db_content['PBD'][pbd_ref]['SR'] = sr_ref + return sr_ref + + +def get_all(table): + return _db_content[table].keys() + + +def get_all_records(table): + return _db_content[table] + + +def get_record(table, ref): + if ref in _db_content[table]: + return _db_content[table].get(ref) + else: + raise Failure(['HANDLE_INVALID', table, ref]) + + +def check_for_session_leaks(): + if len(_db_content['session']) > 0: + raise exception.Error('Sessions have leaked: %s' % + _db_content['session']) + + +class Failure(Exception): + def __init__(self, details): + self.details = details + + def __str__(self): + try: + return str(self.details) + except Exception, exc: + return "XenAPI Fake Failure: %s" % str(self.details) + + def _details_map(self): + return dict([(str(i), self.details[i]) + for i in range(len(self.details))]) + + +class SessionBase(object): + """ + Base class for Fake Sessions + """ + + def __init__(self, uri): + self._session = None + + def xenapi_request(self, methodname, params): + if methodname.startswith('login'): + self._login(methodname, params) + return None + elif methodname == 'logout' or methodname == 'session.logout': + self._logout() + return None + else: + full_params = (self._session,) + params + meth = getattr(self, methodname, None) + if meth is None: + logging.warn('Raising NotImplemented') + raise NotImplementedError( + 'xenapi.fake does not have an implementation for %s' % + methodname) + return meth(*full_params) + + def _login(self, method, params): + self._session = str(uuid.uuid4()) + _db_content['session'][self._session] = { + 'uuid': str(uuid.uuid4()), + 'this_host': _db_content['host'].keys()[0], + } + + def _logout(self): + s = self._session + self._session = None + if s not in _db_content['session']: + raise exception.Error( + "Logging out a session that is invalid or already logged " + "out: %s" % s) + del _db_content['session'][s] + + def __getattr__(self, name): + if name == 'handle': + return self._session + elif name == 'xenapi': + return _Dispatcher(self.xenapi_request, None) + elif name.startswith('login') or name.startswith('slave_local'): + return lambda *params: self._login(name, params) + elif name.startswith('Async'): + return lambda *params: self._async(name, params) + elif '.' in name: + impl = getattr(self, name.replace('.', '_')) + if impl is not None: + def callit(*params): + logging.warn('Calling %s %s', name, impl) + self._check_session(params) + return impl(*params) + return callit + if self._is_gettersetter(name, True): + logging.warn('Calling getter %s', name) + return lambda *params: self._getter(name, params) + elif self._is_create(name): + return lambda *params: self._create(name, params) + else: + return None + + def _is_gettersetter(self, name, getter): + bits = name.split('.') + return (len(bits) == 2 and + bits[0] in _CLASSES and + bits[1].startswith(getter and 'get_' or 'set_')) + + def _is_create(self, name): + bits = name.split('.') + return (len(bits) == 2 and + bits[0] in _CLASSES and + bits[1] == 'create') + + def _getter(self, name, params): + self._check_session(params) + (cls, func) = name.split('.') + + if func == 'get_all': + self._check_arg_count(params, 1) + return get_all(cls) + + if func == 'get_all_records': + self._check_arg_count(params, 1) + return get_all_records(cls) + + if func == 'get_record': + self._check_arg_count(params, 2) + return get_record(cls, params[1]) + + if (func == 'get_by_name_label' or + func == 'get_by_uuid'): + self._check_arg_count(params, 2) + return self._get_by_field( + _db_content[cls], func[len('get_by_'):], params[1]) + + if len(params) == 2: + field = func[len('get_'):] + ref = params[1] + + if (ref in _db_content[cls] and + field in _db_content[cls][ref]): + return _db_content[cls][ref][field] + + logging.error('Raising NotImplemented') + raise NotImplementedError( + 'xenapi.fake does not have an implementation for %s or it has ' + 'been called with the wrong number of arguments' % name) + + def _setter(self, name, params): + self._check_session(params) + (cls, func) = name.split('.') + + if len(params) == 3: + field = func[len('set_'):] + ref = params[1] + val = params[2] + + if (ref in _db_content[cls] and + field in _db_content[cls][ref]): + _db_content[cls][ref][field] = val + + logging.warn('Raising NotImplemented') + raise NotImplementedError( + 'xenapi.fake does not have an implementation for %s or it has ' + 'been called with the wrong number of arguments or the database ' + 'is missing that field' % name) + + def _create(self, name, params): + self._check_session(params) + is_sr_create = name == 'SR.create' + # Storage Repositories have a different API + expected = is_sr_create and 10 or 2 + self._check_arg_count(params, expected) + (cls, _) = name.split('.') + ref = is_sr_create and \ + _create_sr(cls, params) or _create_object(cls, params[1]) + obj = get_record(cls, ref) + + # Add RO fields + if cls == 'VM': + obj['power_state'] = 'Halted' + + return ref + + def _async(self, name, params): + task_ref = create_task(name) + task = _db_content['task'][task_ref] + func = name[len('Async.'):] + try: + task['result'] = self.xenapi_request(func, params[1:]) + task['status'] = 'success' + except Failure, exc: + task['error_info'] = exc.details + task['status'] = 'failed' + task['finished'] = datetime.datetime.now() + return task_ref + + def _check_session(self, params): + if (self._session is None or + self._session not in _db_content['session']): + raise Failure(['HANDLE_INVALID', 'session', self._session]) + if len(params) == 0 or params[0] != self._session: + logging.warn('Raising NotImplemented') + raise NotImplementedError('Call to XenAPI without using .xenapi') + + def _check_arg_count(self, params, expected): + actual = len(params) + if actual != expected: + raise Failure(['MESSAGE_PARAMETER_COUNT_MISMATCH', + expected, actual]) + + def _get_by_field(self, recs, k, v): + result = [] + for ref, rec in recs.iteritems(): + if rec.get(k) == v: + result.append(ref) + return result + + +# Based upon _Method from xmlrpclib. +class _Dispatcher: + def __init__(self, send, name): + self.__send = send + self.__name = name + + def __repr__(self): + if self.__name: + return '<xenapi.fake._Dispatcher for %s>' % self.__name + else: + return '<xenapi.fake._Dispatcher>' + + def __getattr__(self, name): + if self.__name is None: + return _Dispatcher(self.__send, name) + else: + return _Dispatcher(self.__send, "%s.%s" % (self.__name, name)) + + def __call__(self, *args): + return self.__send(self.__name, args) diff --git a/nova/virt/xenapi/network_utils.py b/nova/virt/xenapi/network_utils.py new file mode 100644 index 000000000..c0406d8f0 --- /dev/null +++ b/nova/virt/xenapi/network_utils.py @@ -0,0 +1,43 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Helper methods for operations related to the management of network +records and their attributes like bridges, PIFs, QoS, as well as +their lookup functions. +""" + + +from nova.virt.xenapi import HelperBase + + +class NetworkHelper(HelperBase): + """ + The class that wraps the helper methods together. + """ + + @classmethod + def find_network_with_bridge(cls, session, bridge): + """Return the network on which the bridge is attached, if found.""" + expr = 'field "bridge" = "%s"' % bridge + networks = session.call_xenapi('network.get_all_records_where', expr) + if len(networks) == 1: + return networks.keys()[0] + elif len(networks) > 1: + raise Exception(_('Found non-unique network' + ' for bridge %s') % bridge) + else: + raise Exception(_('Found no network for bridge %s') % bridge) diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py new file mode 100644 index 000000000..89e02c917 --- /dev/null +++ b/nova/virt/xenapi/vm_utils.py @@ -0,0 +1,271 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Helper methods for operations related to the management of VM records and +their attributes like VDIs, VIFs, as well as their lookup functions. +""" + +import logging +import urllib +from xml.dom import minidom + +from nova import exception +from nova import flags +from nova.auth.manager import AuthManager +from nova.compute import instance_types +from nova.compute import power_state +from nova.virt import images +from nova.virt.xenapi import HelperBase +from nova.virt.xenapi.volume_utils import StorageError + + +FLAGS = flags.FLAGS + +XENAPI_POWER_STATE = { + 'Halted': power_state.SHUTDOWN, + 'Running': power_state.RUNNING, + 'Paused': power_state.PAUSED, + 'Suspended': power_state.SHUTDOWN, # FIXME + 'Crashed': power_state.CRASHED} + + +class VMHelper(HelperBase): + """ + The class that wraps the helper methods together. + """ + + @classmethod + def create_vm(cls, session, instance, kernel, ramdisk): + """Create a VM record. Returns a Deferred that gives the new + VM reference.""" + instance_type = instance_types.INSTANCE_TYPES[instance.instance_type] + mem = str(long(instance_type['memory_mb']) * 1024 * 1024) + vcpus = str(instance_type['vcpus']) + rec = { + 'name_label': instance.name, + 'name_description': '', + 'is_a_template': False, + 'memory_static_min': '0', + 'memory_static_max': mem, + 'memory_dynamic_min': mem, + 'memory_dynamic_max': mem, + 'VCPUs_at_startup': vcpus, + 'VCPUs_max': vcpus, + 'VCPUs_params': {}, + 'actions_after_shutdown': 'destroy', + 'actions_after_reboot': 'restart', + 'actions_after_crash': 'destroy', + 'PV_bootloader': '', + 'PV_kernel': kernel, + 'PV_ramdisk': ramdisk, + 'PV_args': 'root=/dev/xvda1', + 'PV_bootloader_args': '', + 'PV_legacy_args': '', + 'HVM_boot_policy': '', + 'HVM_boot_params': {}, + 'platform': {}, + 'PCI_bus': '', + 'recommendations': '', + 'affinity': '', + 'user_version': '0', + 'other_config': {}, + } + logging.debug(_('Created VM %s...'), instance.name) + vm_ref = session.call_xenapi('VM.create', rec) + logging.debug(_('Created VM %s as %s.'), instance.name, vm_ref) + return vm_ref + + @classmethod + def create_vbd(cls, session, vm_ref, vdi_ref, userdevice, bootable): + """Create a VBD record. Returns a Deferred that gives the new + VBD reference.""" + vbd_rec = {} + vbd_rec['VM'] = vm_ref + vbd_rec['VDI'] = vdi_ref + vbd_rec['userdevice'] = str(userdevice) + vbd_rec['bootable'] = bootable + vbd_rec['mode'] = 'RW' + vbd_rec['type'] = 'disk' + vbd_rec['unpluggable'] = True + vbd_rec['empty'] = False + vbd_rec['other_config'] = {} + vbd_rec['qos_algorithm_type'] = '' + vbd_rec['qos_algorithm_params'] = {} + vbd_rec['qos_supported_algorithms'] = [] + logging.debug(_('Creating VBD for VM %s, VDI %s ... '), + vm_ref, vdi_ref) + vbd_ref = session.call_xenapi('VBD.create', vbd_rec) + logging.debug(_('Created VBD %s for VM %s, VDI %s.'), vbd_ref, vm_ref, + vdi_ref) + return vbd_ref + + @classmethod + def find_vbd_by_number(cls, session, vm_ref, number): + """Get the VBD reference from the device number""" + vbds = session.get_xenapi().VM.get_VBDs(vm_ref) + if vbds: + for vbd in vbds: + try: + vbd_rec = session.get_xenapi().VBD.get_record(vbd) + if vbd_rec['userdevice'] == str(number): + return vbd + except cls.XenAPI.Failure, exc: + logging.warn(exc) + raise StorageError(_('VBD not found in instance %s') % vm_ref) + + @classmethod + def unplug_vbd(cls, session, vbd_ref): + """Unplug VBD from VM""" + try: + vbd_ref = session.call_xenapi('VBD.unplug', vbd_ref) + except cls.XenAPI.Failure, exc: + logging.warn(exc) + if exc.details[0] != 'DEVICE_ALREADY_DETACHED': + raise StorageError(_('Unable to unplug VBD %s') % vbd_ref) + + @classmethod + def destroy_vbd(cls, session, vbd_ref): + """Destroy VBD from host database""" + try: + task = session.call_xenapi('Async.VBD.destroy', vbd_ref) + #FIXME(armando): find a solution to missing instance_id + #with Josh Kearney + session.wait_for_task(0, task) + except cls.XenAPI.Failure, exc: + logging.warn(exc) + raise StorageError(_('Unable to destroy VBD %s') % vbd_ref) + + @classmethod + def create_vif(cls, session, vm_ref, network_ref, mac_address): + """Create a VIF record. Returns a Deferred that gives the new + VIF reference.""" + vif_rec = {} + vif_rec['device'] = '0' + vif_rec['network'] = network_ref + vif_rec['VM'] = vm_ref + vif_rec['MAC'] = mac_address + vif_rec['MTU'] = '1500' + vif_rec['other_config'] = {} + vif_rec['qos_algorithm_type'] = '' + vif_rec['qos_algorithm_params'] = {} + logging.debug(_('Creating VIF for VM %s, network %s.'), vm_ref, + network_ref) + vif_ref = session.call_xenapi('VIF.create', vif_rec) + logging.debug(_('Created VIF %s for VM %s, network %s.'), vif_ref, + vm_ref, network_ref) + return vif_ref + + @classmethod + def fetch_image(cls, session, image, user, project, use_sr): + """use_sr: True to put the image as a VDI in an SR, False to place + it on dom0's filesystem. The former is for VM disks, the latter for + its kernel and ramdisk (if external kernels are being used). + Returns a Deferred that gives the new VDI UUID.""" + + url = images.image_url(image) + access = AuthManager().get_access_key(user, project) + logging.debug(_("Asking xapi to fetch %s as %s"), url, access) + fn = use_sr and 'get_vdi' or 'get_kernel' + args = {} + args['src_url'] = url + args['username'] = access + args['password'] = user.secret + if use_sr: + args['add_partition'] = 'true' + task = session.async_call_plugin('objectstore', fn, args) + #FIXME(armando): find a solution to missing instance_id + #with Josh Kearney + uuid = session.wait_for_task(0, task) + return uuid + + @classmethod + def lookup(cls, session, i): + """Look the instance i up, and returns it if available""" + vms = session.get_xenapi().VM.get_by_name_label(i) + n = len(vms) + if n == 0: + return None + elif n > 1: + raise exception.Duplicate(_('duplicate name found: %s') % i) + else: + return vms[0] + + @classmethod + def lookup_vm_vdis(cls, session, vm): + """Look for the VDIs that are attached to the VM""" + # Firstly we get the VBDs, then the VDIs. + # TODO(Armando): do we leave the read-only devices? + vbds = session.get_xenapi().VM.get_VBDs(vm) + vdis = [] + if vbds: + for vbd in vbds: + try: + vdi = session.get_xenapi().VBD.get_VDI(vbd) + # Test valid VDI + record = session.get_xenapi().VDI.get_record(vdi) + logging.debug(_('VDI %s is still available'), + record['uuid']) + except cls.XenAPI.Failure, exc: + logging.warn(exc) + else: + vdis.append(vdi) + if len(vdis) > 0: + return vdis + else: + return None + + @classmethod + def compile_info(cls, record): + """Fill record with VM status information""" + return {'state': XENAPI_POWER_STATE[record['power_state']], + 'max_mem': long(record['memory_static_max']) >> 10, + 'mem': long(record['memory_dynamic_max']) >> 10, + 'num_cpu': record['VCPUs_max'], + 'cpu_time': 0} + + @classmethod + def compile_diagnostics(cls, session, record): + """Compile VM diagnostics data""" + try: + host = session.get_xenapi_host() + host_ip = session.get_xenapi().host.get_record(host)["address"] + diags = {} + xml = get_rrd(host_ip, record["uuid"]) + if xml: + rrd = minidom.parseString(xml) + for i, node in enumerate(rrd.firstChild.childNodes): + # We don't want all of the extra garbage + if i >= 3 and i <= 11: + ref = node.childNodes + # Name and Value + diags[ref[0].firstChild.data] = ref[6].firstChild.data + return diags + except cls.XenAPI.Failure as e: + return {"Unable to retrieve diagnostics": e} + + +def get_rrd(host, uuid): + """Return the VM RRD XML as a string""" + try: + xml = urllib.urlopen("http://%s:%s@%s/vm_rrd?uuid=%s" % ( + FLAGS.xenapi_connection_username, + FLAGS.xenapi_connection_password, + host, + uuid)) + return xml.read() + except IOError: + return None diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py new file mode 100644 index 000000000..39bb0f1e8 --- /dev/null +++ b/nova/virt/xenapi/vmops.py @@ -0,0 +1,196 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Management class for VM-related functions (spawn, reboot, etc). +""" + +import logging + +from nova import db +from nova import context +from nova import exception +from nova import utils + +from nova.auth.manager import AuthManager +from nova.compute import power_state +from nova.virt.xenapi.network_utils import NetworkHelper +from nova.virt.xenapi.vm_utils import VMHelper + + +class VMOps(object): + """ + Management class for VM-related tasks + """ + + def __init__(self, session): + self.XenAPI = session.get_imported_xenapi() + self._session = session + VMHelper.XenAPI = self.XenAPI + + def list_instances(self): + """List VM instances""" + vms = [] + for vm in self._session.get_xenapi().VM.get_all(): + rec = self._session.get_xenapi().VM.get_record(vm) + if not rec["is_a_template"] and not rec["is_control_domain"]: + vms.append(rec["name_label"]) + return vms + + def spawn(self, instance): + """Create VM instance""" + vm = VMHelper.lookup(self._session, instance.name) + if vm is not None: + raise exception.Duplicate(_('Attempted to create' + ' non-unique name %s') % instance.name) + + bridge = db.network_get_by_instance(context.get_admin_context(), + instance['id'])['bridge'] + network_ref = \ + NetworkHelper.find_network_with_bridge(self._session, bridge) + + user = AuthManager().get_user(instance.user_id) + project = AuthManager().get_project(instance.project_id) + vdi_uuid = VMHelper.fetch_image( + self._session, instance.image_id, user, project, True) + kernel = VMHelper.fetch_image( + self._session, instance.kernel_id, user, project, False) + ramdisk = VMHelper.fetch_image( + self._session, instance.ramdisk_id, user, project, False) + vdi_ref = self._session.call_xenapi('VDI.get_by_uuid', vdi_uuid) + vm_ref = VMHelper.create_vm( + self._session, instance, kernel, ramdisk) + VMHelper.create_vbd(self._session, vm_ref, vdi_ref, 0, True) + if network_ref: + VMHelper.create_vif(self._session, vm_ref, + network_ref, instance.mac_address) + logging.debug(_('Starting VM %s...'), vm_ref) + self._session.call_xenapi('VM.start', vm_ref, False, False) + logging.info(_('Spawning VM %s created %s.'), instance.name, + vm_ref) + + # NOTE(armando): Do we really need to do this in virt? + timer = utils.LoopingCall(f=None) + + def _wait_for_boot(): + try: + state = self.get_info(instance['name'])['state'] + db.instance_set_state(context.get_admin_context(), + instance['id'], state) + if state == power_state.RUNNING: + logging.debug(_('Instance %s: booted'), instance['name']) + timer.stop() + except Exception, exc: + logging.warn(exc) + logging.exception(_('instance %s: failed to boot'), + instance['name']) + db.instance_set_state(context.get_admin_context(), + instance['id'], + power_state.SHUTDOWN) + timer.stop() + + timer.f = _wait_for_boot + return timer.start(interval=0.5, now=True) + + def reboot(self, instance): + """Reboot VM instance""" + instance_name = instance.name + vm = VMHelper.lookup(self._session, instance_name) + if vm is None: + raise exception.NotFound(_('instance not' + ' found %s') % instance_name) + task = self._session.call_xenapi('Async.VM.clean_reboot', vm) + self._session.wait_for_task(instance.id, task) + + def destroy(self, instance): + """Destroy VM instance""" + vm = VMHelper.lookup(self._session, instance.name) + if vm is None: + # Don't complain, just return. This lets us clean up instances + # that have already disappeared from the underlying platform. + return + # Get the VDIs related to the VM + vdis = VMHelper.lookup_vm_vdis(self._session, vm) + try: + task = self._session.call_xenapi('Async.VM.hard_shutdown', + vm) + self._session.wait_for_task(instance.id, task) + except XenAPI.Failure, exc: + logging.warn(exc) + # Disk clean-up + if vdis: + for vdi in vdis: + try: + task = self._session.call_xenapi('Async.VDI.destroy', vdi) + self._session.wait_for_task(instance.id, task) + except XenAPI.Failure, exc: + logging.warn(exc) + # VM Destroy + try: + task = self._session.call_xenapi('Async.VM.destroy', vm) + self._session.wait_for_task(instance.id, task) + except XenAPI.Failure, exc: + logging.warn(exc) + + def _wait_with_callback(self, instance_id, task, callback): + ret = None + try: + ret = self._session.wait_for_task(instance_id, task) + except XenAPI.Failure, exc: + logging.warn(exc) + callback(ret) + + def pause(self, instance, callback): + """Pause VM instance""" + instance_name = instance.name + vm = VMHelper.lookup(self._session, instance_name) + if vm is None: + raise exception.NotFound(_('Instance not' + ' found %s') % instance_name) + task = self._session.call_xenapi('Async.VM.pause', vm) + self._wait_with_callback(instance.id, task, callback) + + def unpause(self, instance, callback): + """Unpause VM instance""" + instance_name = instance.name + vm = VMHelper.lookup(self._session, instance_name) + if vm is None: + raise exception.NotFound(_('Instance not' + ' found %s') % instance_name) + task = self._session.call_xenapi('Async.VM.unpause', vm) + self._wait_with_callback(instance.id, task, callback) + + def get_info(self, instance_id): + """Return data about VM instance""" + vm = VMHelper.lookup(self._session, instance_id) + if vm is None: + raise exception.NotFound(_('Instance not' + ' found %s') % instance_id) + rec = self._session.get_xenapi().VM.get_record(vm) + return VMHelper.compile_info(rec) + + def get_diagnostics(self, instance_id): + """Return data about VM diagnostics""" + vm = VMHelper.lookup(self._session, instance_id) + if vm is None: + raise exception.NotFound(_("Instance not found %s") % instance_id) + rec = self._session.get_xenapi().VM.get_record(vm) + return VMHelper.compile_diagnostics(self._session, rec) + + def get_console_output(self, instance): + """Return snapshot of console""" + # TODO: implement this to fix pylint! + return 'FAKE CONSOLE OUTPUT of instance' diff --git a/nova/virt/xenapi/volume_utils.py b/nova/virt/xenapi/volume_utils.py new file mode 100644 index 000000000..a0c0a67d4 --- /dev/null +++ b/nova/virt/xenapi/volume_utils.py @@ -0,0 +1,268 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Helper methods for operations related to the management of volumes, +and storage repositories +""" + +import re +import string +import logging + +from nova import db +from nova import context +from nova import exception +from nova import flags +from nova import utils +from nova.virt.xenapi import HelperBase + +FLAGS = flags.FLAGS + + +class StorageError(Exception): + """To raise errors related to SR, VDI, PBD, and VBD commands""" + + def __init__(self, message=None): + super(StorageError, self).__init__(message) + + +class VolumeHelper(HelperBase): + """ + The class that wraps the helper methods together. + """ + + @classmethod + def create_iscsi_storage(cls, session, info, label, description): + """ + Create an iSCSI storage repository that will be used to mount + the volume for the specified instance + """ + sr_ref = session.get_xenapi().SR.get_by_name_label(label) + if len(sr_ref) == 0: + logging.debug('Introducing %s...', label) + record = {} + if 'chapuser' in info and 'chappassword' in info: + record = {'target': info['targetHost'], + 'port': info['targetPort'], + 'targetIQN': info['targetIQN'], + 'chapuser': info['chapuser'], + 'chappassword': info['chappassword'] + } + else: + record = {'target': info['targetHost'], + 'port': info['targetPort'], + 'targetIQN': info['targetIQN'] + } + try: + sr_ref = session.get_xenapi().SR.create( + session.get_xenapi_host(), + record, + '0', label, description, 'iscsi', '', False, {}) + logging.debug('Introduced %s as %s.', label, sr_ref) + return sr_ref + except cls.XenAPI.Failure, exc: + logging.warn(exc) + raise StorageError(_('Unable to create Storage Repository')) + else: + return sr_ref[0] + + @classmethod + def find_sr_from_vbd(cls, session, vbd_ref): + """Find the SR reference from the VBD reference""" + try: + vdi_ref = session.get_xenapi().VBD.get_VDI(vbd_ref) + sr_ref = session.get_xenapi().VDI.get_SR(vdi_ref) + except cls.XenAPI.Failure, exc: + logging.warn(exc) + raise StorageError(_('Unable to find SR from VBD %s') % vbd_ref) + return sr_ref + + @classmethod + def destroy_iscsi_storage(cls, session, sr_ref): + """Forget the SR whilst preserving the state of the disk""" + logging.debug("Forgetting SR %s ... ", sr_ref) + pbds = [] + try: + pbds = session.get_xenapi().SR.get_PBDs(sr_ref) + except cls.XenAPI.Failure, exc: + logging.warn('Ignoring exception %s when getting PBDs for %s', + exc, sr_ref) + for pbd in pbds: + try: + session.get_xenapi().PBD.unplug(pbd) + except cls.XenAPI.Failure, exc: + logging.warn('Ignoring exception %s when unplugging PBD %s', + exc, pbd) + try: + session.get_xenapi().SR.forget(sr_ref) + logging.debug("Forgetting SR %s done.", sr_ref) + except cls.XenAPI.Failure, exc: + logging.warn('Ignoring exception %s when forgetting SR %s', + exc, sr_ref) + + @classmethod + def introduce_vdi(cls, session, sr_ref): + """Introduce VDI in the host""" + try: + vdis = session.get_xenapi().SR.get_VDIs(sr_ref) + except cls.XenAPI.Failure, exc: + logging.warn(exc) + raise StorageError(_('Unable to introduce VDI on SR %s') % sr_ref) + try: + vdi_rec = session.get_xenapi().VDI.get_record(vdis[0]) + except cls.XenAPI.Failure, exc: + logging.warn(exc) + raise StorageError(_('Unable to get record' + ' of VDI %s on') % vdis[0]) + else: + try: + return session.get_xenapi().VDI.introduce( + vdi_rec['uuid'], + vdi_rec['name_label'], + vdi_rec['name_description'], + vdi_rec['SR'], + vdi_rec['type'], + vdi_rec['sharable'], + vdi_rec['read_only'], + vdi_rec['other_config'], + vdi_rec['location'], + vdi_rec['xenstore_data'], + vdi_rec['sm_config']) + except cls.XenAPI.Failure, exc: + logging.warn(exc) + raise StorageError(_('Unable to introduce VDI for SR %s') + % sr_ref) + + @classmethod + def parse_volume_info(cls, device_path, mountpoint): + """ + Parse device_path and mountpoint as they can be used by XenAPI. + In particular, the mountpoint (e.g. /dev/sdc) must be translated + into a numeric literal. + FIXME(armando): + As for device_path, currently cannot be used as it is, + because it does not contain target information. As for interim + solution, target details are passed either via Flags or obtained + by iscsiadm. Long-term solution is to add a few more fields to the + db in the iscsi_target table with the necessary info and modify + the iscsi driver to set them. + """ + device_number = VolumeHelper.mountpoint_to_number(mountpoint) + volume_id = _get_volume_id(device_path) + (iscsi_name, iscsi_portal) = _get_target(volume_id) + target_host = _get_target_host(iscsi_portal) + target_port = _get_target_port(iscsi_portal) + target_iqn = _get_iqn(iscsi_name, volume_id) + logging.debug('(vol_id,number,host,port,iqn): (%s,%s,%s,%s)', + volume_id, + target_host, + target_port, + target_iqn) + if (device_number < 0) or \ + (volume_id is None) or \ + (target_host is None) or \ + (target_iqn is None): + raise StorageError(_('Unable to obtain target information %s, %s') + % (device_path, mountpoint)) + volume_info = {} + volume_info['deviceNumber'] = device_number + volume_info['volumeId'] = volume_id + volume_info['targetHost'] = target_host + volume_info['targetPort'] = target_port + volume_info['targetIQN'] = target_iqn + return volume_info + + @classmethod + def mountpoint_to_number(cls, mountpoint): + """Translate a mountpoint like /dev/sdc into a numeric""" + if mountpoint.startswith('/dev/'): + mountpoint = mountpoint[5:] + if re.match('^[hs]d[a-p]$', mountpoint): + return (ord(mountpoint[2:3]) - ord('a')) + elif re.match('^vd[a-p]$', mountpoint): + return (ord(mountpoint[2:3]) - ord('a')) + elif re.match('^[0-9]+$', mountpoint): + return string.atoi(mountpoint, 10) + else: + logging.warn('Mountpoint cannot be translated: %s', mountpoint) + return -1 + + +def _get_volume_id(path): + """Retrieve the volume id from device_path""" + # n must contain at least the volume_id + # /vol- is for remote volumes + # -vol- is for local volumes + # see compute/manager->setup_compute_volume + volume_id = path[path.find('/vol-') + 1:] + if volume_id == path: + volume_id = path[path.find('-vol-') + 1:].replace('--', '-') + return volume_id + + +def _get_target_host(iscsi_string): + """Retrieve target host""" + if iscsi_string: + return iscsi_string[0:iscsi_string.find(':')] + elif iscsi_string is None or FLAGS.target_host: + return FLAGS.target_host + + +def _get_target_port(iscsi_string): + """Retrieve target port""" + if iscsi_string: + return iscsi_string[iscsi_string.find(':') + 1:] + elif iscsi_string is None or FLAGS.target_port: + return FLAGS.target_port + + +def _get_iqn(iscsi_string, id): + """Retrieve target IQN""" + if iscsi_string: + return iscsi_string + elif iscsi_string is None or FLAGS.iqn_prefix: + volume_id = _get_volume_id(id) + return '%s:%s' % (FLAGS.iqn_prefix, volume_id) + + +def _get_target(volume_id): + """ + Gets iscsi name and portal from volume name and host. + For this method to work the following are needed: + 1) volume_ref['host'] must resolve to something rather than loopback + 2) ietd must bind only to the address as resolved above + If any of the two conditions are not met, fall back on Flags. + """ + volume_ref = db.volume_get_by_ec2_id(context.get_admin_context(), + volume_id) + result = (None, None) + try: + (r, _e) = utils.execute("sudo iscsiadm -m discovery -t " + "sendtargets -p %s" % + volume_ref['host']) + except exception.ProcessExecutionError, exc: + logging.warn(exc) + else: + targets = r.splitlines() + if len(_e) == 0 and len(targets) == 1: + for target in targets: + if volume_id in target: + (location, _sep, iscsi_name) = target.partition(" ") + break + iscsi_portal = location.split(",")[0] + result = (iscsi_name, iscsi_portal) + return result diff --git a/nova/virt/xenapi/volumeops.py b/nova/virt/xenapi/volumeops.py new file mode 100644 index 000000000..fdeb2506c --- /dev/null +++ b/nova/virt/xenapi/volumeops.py @@ -0,0 +1,126 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Management class for Storage-related functions (attach, detach, etc). +""" +import logging + +from nova import exception +from nova.virt.xenapi.vm_utils import VMHelper +from nova.virt.xenapi.volume_utils import VolumeHelper +from nova.virt.xenapi.volume_utils import StorageError + + +class VolumeOps(object): + """ + Management class for Volume-related tasks + """ + + def __init__(self, session): + self.XenAPI = session.get_imported_xenapi() + self._session = session + # Load XenAPI module in the helper classes respectively + VolumeHelper.XenAPI = self.XenAPI + VMHelper.XenAPI = self.XenAPI + + def attach_volume(self, instance_name, device_path, mountpoint): + """Attach volume storage to VM instance""" + # Before we start, check that the VM exists + vm_ref = VMHelper.lookup(self._session, instance_name) + if vm_ref is None: + raise exception.NotFound(_('Instance %s not found') + % instance_name) + # NOTE: No Resource Pool concept so far + logging.debug(_("Attach_volume: %s, %s, %s"), + instance_name, device_path, mountpoint) + # Create the iSCSI SR, and the PDB through which hosts access SRs. + # But first, retrieve target info, like Host, IQN, LUN and SCSIID + vol_rec = VolumeHelper.parse_volume_info(device_path, mountpoint) + label = 'SR-%s' % vol_rec['volumeId'] + description = 'Disk-for:%s' % instance_name + # Create SR + sr_ref = VolumeHelper.create_iscsi_storage(self._session, + vol_rec, + label, + description) + # Introduce VDI and attach VBD to VM + try: + vdi_ref = VolumeHelper.introduce_vdi(self._session, sr_ref) + except StorageError, exc: + logging.warn(exc) + VolumeHelper.destroy_iscsi_storage(self._session, sr_ref) + raise Exception(_('Unable to create VDI on SR %s for instance %s') + % (sr_ref, + instance_name)) + else: + try: + vbd_ref = VMHelper.create_vbd(self._session, + vm_ref, vdi_ref, + vol_rec['deviceNumber'], + False) + except self.XenAPI.Failure, exc: + logging.warn(exc) + VolumeHelper.destroy_iscsi_storage(self._session, sr_ref) + raise Exception(_('Unable to use SR %s for instance %s') + % (sr_ref, + instance_name)) + else: + try: + task = self._session.call_xenapi('Async.VBD.plug', + vbd_ref) + self._session.wait_for_task(vol_rec['deviceNumber'], task) + except self.XenAPI.Failure, exc: + logging.warn(exc) + VolumeHelper.destroy_iscsi_storage(self._session, + sr_ref) + raise Exception(_('Unable to attach volume to instance %s') + % instance_name) + logging.info(_('Mountpoint %s attached to instance %s'), + mountpoint, instance_name) + + def detach_volume(self, instance_name, mountpoint): + """Detach volume storage to VM instance""" + # Before we start, check that the VM exists + vm_ref = VMHelper.lookup(self._session, instance_name) + if vm_ref is None: + raise exception.NotFound(_('Instance %s not found') + % instance_name) + # Detach VBD from VM + logging.debug(_("Detach_volume: %s, %s"), instance_name, mountpoint) + device_number = VolumeHelper.mountpoint_to_number(mountpoint) + try: + vbd_ref = VMHelper.find_vbd_by_number(self._session, + vm_ref, device_number) + except StorageError, exc: + logging.warn(exc) + raise Exception(_('Unable to locate volume %s') % mountpoint) + else: + try: + sr_ref = VolumeHelper.find_sr_from_vbd(self._session, + vbd_ref) + VMHelper.unplug_vbd(self._session, vbd_ref) + except StorageError, exc: + logging.warn(exc) + raise Exception(_('Unable to detach volume %s') % mountpoint) + try: + VMHelper.destroy_vbd(self._session, vbd_ref) + except StorageError, exc: + logging.warn(exc) + # Forget SR + VolumeHelper.destroy_iscsi_storage(self._session, sr_ref) + logging.info(_('Mountpoint %s detached from instance %s'), + mountpoint, instance_name) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py new file mode 100644 index 000000000..11c66c974 --- /dev/null +++ b/nova/virt/xenapi_conn.py @@ -0,0 +1,285 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2010 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +A connection to XenServer or Xen Cloud Platform. + +The concurrency model for this class is as follows: + +All XenAPI calls are on a thread (using t.i.t.deferToThread, via the decorator +deferredToThread). They are remote calls, and so may hang for the usual +reasons. They should not be allowed to block the reactor thread. + +All long-running XenAPI calls (VM.start, VM.reboot, etc) are called async +(using XenAPI.VM.async_start etc). These return a task, which can then be +polled for completion. Polling is handled using reactor.callLater. + +This combination of techniques means that we don't block the reactor thread at +all, and at the same time we don't hold lots of threads waiting for +long-running operations. + +FIXME: get_info currently doesn't conform to these rules, and will block the +reactor thread if the VM.get_by_name_label or VM.get_record calls block. + +**Related Flags** + +:xenapi_connection_url: URL for connection to XenServer/Xen Cloud Platform. +:xenapi_connection_username: Username for connection to XenServer/Xen Cloud + Platform (default: root). +:xenapi_connection_password: Password for connection to XenServer/Xen Cloud + Platform. +:xenapi_task_poll_interval: The interval (seconds) used for polling of + remote tasks (Async.VM.start, etc) + (default: 0.5). +:target_host: the iSCSI Target Host IP address, i.e. the IP + address for the nova-volume host +:target_port: iSCSI Target Port, 3260 Default +:iqn_prefix: IQN Prefix, e.g. 'iqn.2010-10.org.openstack' +""" + +import logging +import sys +import xmlrpclib + +from eventlet import event +from eventlet import tpool + +from nova import context +from nova import db +from nova import utils +from nova import flags +from nova.virt.xenapi.vmops import VMOps +from nova.virt.xenapi.volumeops import VolumeOps + +FLAGS = flags.FLAGS + +flags.DEFINE_string('xenapi_connection_url', + None, + 'URL for connection to XenServer/Xen Cloud Platform.' + ' Required if connection_type=xenapi.') +flags.DEFINE_string('xenapi_connection_username', + 'root', + 'Username for connection to XenServer/Xen Cloud Platform.' + ' Used only if connection_type=xenapi.') +flags.DEFINE_string('xenapi_connection_password', + None, + 'Password for connection to XenServer/Xen Cloud Platform.' + ' Used only if connection_type=xenapi.') +flags.DEFINE_float('xenapi_task_poll_interval', + 0.5, + 'The interval used for polling of remote tasks ' + '(Async.VM.start, etc). Used only if ' + 'connection_type=xenapi.') +flags.DEFINE_string('target_host', + None, + 'iSCSI Target Host') +flags.DEFINE_string('target_port', + '3260', + 'iSCSI Target Port, 3260 Default') +flags.DEFINE_string('iqn_prefix', + 'iqn.2010-10.org.openstack', + 'IQN Prefix') + + +def get_connection(_): + """Note that XenAPI doesn't have a read-only connection mode, so + the read_only parameter is ignored.""" + url = FLAGS.xenapi_connection_url + username = FLAGS.xenapi_connection_username + password = FLAGS.xenapi_connection_password + if not url or password is None: + raise Exception(_('Must specify xenapi_connection_url, ' + 'xenapi_connection_username (optionally), and ' + 'xenapi_connection_password to use ' + 'connection_type=xenapi')) + return XenAPIConnection(url, username, password) + + +class XenAPIConnection(object): + """A connection to XenServer or Xen Cloud Platform""" + + def __init__(self, url, user, pw): + session = XenAPISession(url, user, pw) + self._vmops = VMOps(session) + self._volumeops = VolumeOps(session) + + def init_host(self): + #FIXME(armando): implement this + #NOTE(armando): would we need a method + #to call when shutting down the host? + #e.g. to do session logout? + pass + + def list_instances(self): + """List VM instances""" + return self._vmops.list_instances() + + def spawn(self, instance): + """Create VM instance""" + self._vmops.spawn(instance) + + def reboot(self, instance): + """Reboot VM instance""" + self._vmops.reboot(instance) + + def destroy(self, instance): + """Destroy VM instance""" + self._vmops.destroy(instance) + + def pause(self, instance, callback): + """Pause VM instance""" + self._vmops.pause(instance, callback) + + def unpause(self, instance, callback): + """Unpause paused VM instance""" + self._vmops.unpause(instance, callback) + + def get_info(self, instance_id): + """Return data about VM instance""" + return self._vmops.get_info(instance_id) + + def get_diagnostics(self, instance_id): + """Return data about VM diagnostics""" + return self._vmops.get_diagnostics(instance_id) + + def get_console_output(self, instance): + """Return snapshot of console""" + return self._vmops.get_console_output(instance) + + def attach_volume(self, instance_name, device_path, mountpoint): + """Attach volume storage to VM instance""" + return self._volumeops.attach_volume(instance_name, + device_path, + mountpoint) + + def detach_volume(self, instance_name, mountpoint): + """Detach volume storage to VM instance""" + return self._volumeops.detach_volume(instance_name, mountpoint) + + +class XenAPISession(object): + """The session to invoke XenAPI SDK calls""" + + def __init__(self, url, user, pw): + self.XenAPI = self.get_imported_xenapi() + self._session = self._create_session(url) + self._session.login_with_password(user, pw) + + def get_imported_xenapi(self): + """Stubout point. This can be replaced with a mock xenapi module.""" + return __import__('XenAPI') + + def get_xenapi(self): + """Return the xenapi object""" + return self._session.xenapi + + def get_xenapi_host(self): + """Return the xenapi host""" + return self._session.xenapi.session.get_this_host(self._session.handle) + + def call_xenapi(self, method, *args): + """Call the specified XenAPI method on a background thread.""" + f = self._session.xenapi + for m in method.split('.'): + f = f.__getattr__(m) + return tpool.execute(f, *args) + + def async_call_plugin(self, plugin, fn, args): + """Call Async.host.call_plugin on a background thread.""" + return tpool.execute(self._unwrap_plugin_exceptions, + self._session.xenapi.Async.host.call_plugin, + self.get_xenapi_host(), plugin, fn, args) + + def wait_for_task(self, id, task): + """Return the result of the given task. The task is polled + until it completes.""" + + done = event.Event() + loop = utils.LoopingCall(self._poll_task, id, task, done) + loop.start(FLAGS.xenapi_task_poll_interval, now=True) + rv = done.wait() + loop.stop() + return rv + + def _create_session(self, url): + """Stubout point. This can be replaced with a mock session.""" + return self.XenAPI.Session(url) + + def _poll_task(self, id, task, done): + """Poll the given XenAPI task, and fire the given Deferred if we + get a result.""" + try: + name = self._session.xenapi.task.get_name_label(task) + status = self._session.xenapi.task.get_status(task) + action = dict( + id=int(id), + action=name, + error=None) + if status == "pending": + return + elif status == "success": + result = self._session.xenapi.task.get_result(task) + logging.info(_("Task [%s] %s status: success %s") % ( + name, + task, + result)) + done.send(_parse_xmlrpc_value(result)) + else: + error_info = self._session.xenapi.task.get_error_info(task) + action["error"] = str(error_info) + logging.warn(_("Task [%s] %s status: %s %s") % ( + name, + task, + status, + error_info)) + done.send_exception(self.XenAPI.Failure(error_info)) + db.instance_action_create(context.get_admin_context(), action) + except self.XenAPI.Failure, exc: + logging.warn(exc) + done.send_exception(*sys.exc_info()) + + def _unwrap_plugin_exceptions(self, func, *args, **kwargs): + """Parse exception details""" + try: + return func(*args, **kwargs) + except self.XenAPI.Failure, exc: + logging.debug(_("Got exception: %s"), exc) + if (len(exc.details) == 4 and + exc.details[0] == 'XENAPI_PLUGIN_EXCEPTION' and + exc.details[2] == 'Failure'): + params = None + try: + params = eval(exc.details[3]) + except: + raise exc + raise self.XenAPI.Failure(params) + else: + raise + except xmlrpclib.ProtocolError, exc: + logging.debug(_("Got exception: %s"), exc) + raise + + +def _parse_xmlrpc_value(val): + """Parse the given value as if it were an XML-RPC value. This is + sometimes used as the format for the task.result field.""" + if not val: + return val + x = xmlrpclib.loads( + '<?xml version="1.0"?><methodResponse><params><param>' + + val + + '</param></params></methodResponse>') + return x[0][0] diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 156aad2a0..8353b9712 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -22,12 +22,10 @@ Drivers for volumes. import logging import os - -from twisted.internet import defer +import time from nova import exception from nova import flags -from nova import process from nova import utils @@ -55,14 +53,13 @@ flags.DEFINE_string('iscsi_ip_prefix', '127.0', class VolumeDriver(object): """Executes commands relating to Volumes.""" - def __init__(self, execute=process.simple_execute, + def __init__(self, execute=utils.execute, sync_exec=utils.execute, *args, **kwargs): # NOTE(vish): db is set by Manager self.db = None self._execute = execute self._sync_exec = sync_exec - @defer.inlineCallbacks def _try_execute(self, command): # NOTE(vish): Volume commands can partially fail due to timing, but # running them a second time on failure will usually @@ -70,69 +67,61 @@ class VolumeDriver(object): tries = 0 while True: try: - yield self._execute(command) - defer.returnValue(True) + self._execute(command) + return True except exception.ProcessExecutionError: tries = tries + 1 if tries >= FLAGS.num_shell_tries: raise - logging.exception("Recovering from a failed execute." - "Try number %s", tries) - yield self._execute("sleep %s" % tries ** 2) + logging.exception(_("Recovering from a failed execute." + "Try number %s"), tries) + time.sleep(tries ** 2) def check_for_setup_error(self): """Returns an error if prerequisites aren't met""" if not os.path.isdir("/dev/%s" % FLAGS.volume_group): - raise exception.Error("volume group %s doesn't exist" + raise exception.Error(_("volume group %s doesn't exist") % FLAGS.volume_group) - @defer.inlineCallbacks def create_volume(self, volume): """Creates a logical volume.""" if int(volume['size']) == 0: sizestr = '100M' else: sizestr = '%sG' % volume['size'] - yield self._try_execute("sudo lvcreate -L %s -n %s %s" % - (sizestr, - volume['name'], - FLAGS.volume_group)) + self._try_execute("sudo lvcreate -L %s -n %s %s" % + (sizestr, + volume['name'], + FLAGS.volume_group)) - @defer.inlineCallbacks def delete_volume(self, volume): """Deletes a logical volume.""" - yield self._try_execute("sudo lvremove -f %s/%s" % - (FLAGS.volume_group, - volume['name'])) + self._try_execute("sudo lvremove -f %s/%s" % + (FLAGS.volume_group, + volume['name'])) - @defer.inlineCallbacks def local_path(self, volume): - yield # NOTE(vish): stops deprecation warning + # NOTE(vish): stops deprecation warning escaped_group = FLAGS.volume_group.replace('-', '--') escaped_name = volume['name'].replace('-', '--') - defer.returnValue("/dev/mapper/%s-%s" % (escaped_group, - escaped_name)) + return "/dev/mapper/%s-%s" % (escaped_group, escaped_name) def ensure_export(self, context, volume): """Synchronously recreates an export for a logical volume.""" raise NotImplementedError() - @defer.inlineCallbacks def create_export(self, context, volume): """Exports the volume.""" raise NotImplementedError() - @defer.inlineCallbacks def remove_export(self, context, volume): """Removes an export for a logical volume.""" raise NotImplementedError() - @defer.inlineCallbacks def discover_volume(self, volume): """Discover volume on a remote host.""" raise NotImplementedError() - @defer.inlineCallbacks def undiscover_volume(self, volume): """Undiscover volume on a remote host.""" raise NotImplementedError() @@ -155,14 +144,13 @@ class AOEDriver(VolumeDriver): dev = {'shelf_id': shelf_id, 'blade_id': blade_id} self.db.export_device_create_safe(context, dev) - @defer.inlineCallbacks def create_export(self, context, volume): """Creates an export for a logical volume.""" self._ensure_blades(context) (shelf_id, blade_id) = self.db.volume_allocate_shelf_and_blade(context, volume['id']) - yield self._try_execute( + self._try_execute( "sudo vblade-persist setup %s %s %s /dev/%s/%s" % (shelf_id, blade_id, @@ -176,33 +164,30 @@ class AOEDriver(VolumeDriver): # still works for the other volumes, so we # just wait a bit for the current volume to # be ready and ignore any errors. - yield self._execute("sleep 2") - yield self._execute("sudo vblade-persist auto all", - check_exit_code=False) - yield self._execute("sudo vblade-persist start all", - check_exit_code=False) + time.sleep(2) + self._execute("sudo vblade-persist auto all", + check_exit_code=False) + self._execute("sudo vblade-persist start all", + check_exit_code=False) - @defer.inlineCallbacks def remove_export(self, context, volume): """Removes an export for a logical volume.""" (shelf_id, blade_id) = self.db.volume_get_shelf_and_blade(context, volume['id']) - yield self._try_execute("sudo vblade-persist stop %s %s" % - (shelf_id, blade_id)) - yield self._try_execute("sudo vblade-persist destroy %s %s" % - (shelf_id, blade_id)) + self._try_execute("sudo vblade-persist stop %s %s" % + (shelf_id, blade_id)) + self._try_execute("sudo vblade-persist destroy %s %s" % + (shelf_id, blade_id)) - @defer.inlineCallbacks def discover_volume(self, _volume): """Discover volume on a remote host.""" - yield self._execute("sudo aoe-discover") - yield self._execute("sudo aoe-stat", check_exit_code=False) + self._execute("sudo aoe-discover") + self._execute("sudo aoe-stat", check_exit_code=False) - @defer.inlineCallbacks def undiscover_volume(self, _volume): """Undiscover volume on a remote host.""" - yield + pass class FakeAOEDriver(AOEDriver): @@ -220,7 +205,7 @@ class FakeAOEDriver(AOEDriver): @staticmethod def fake_execute(cmd, *_args, **_kwargs): """Execute that simply logs the command.""" - logging.debug("FAKE AOE: %s", cmd) + logging.debug(_("FAKE AOE: %s"), cmd) return (None, None) @@ -252,7 +237,6 @@ class ISCSIDriver(VolumeDriver): target = {'host': host, 'target_num': target_num} self.db.iscsi_target_create_safe(context, target) - @defer.inlineCallbacks def create_export(self, context, volume): """Creates an export for a logical volume.""" self._ensure_iscsi_targets(context, volume['host']) @@ -261,61 +245,55 @@ class ISCSIDriver(VolumeDriver): volume['host']) iscsi_name = "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) volume_path = "/dev/%s/%s" % (FLAGS.volume_group, volume['name']) - yield self._execute("sudo ietadm --op new " - "--tid=%s --params Name=%s" % - (iscsi_target, iscsi_name)) - yield self._execute("sudo ietadm --op new --tid=%s " - "--lun=0 --params Path=%s,Type=fileio" % - (iscsi_target, volume_path)) - - @defer.inlineCallbacks + self._execute("sudo ietadm --op new " + "--tid=%s --params Name=%s" % + (iscsi_target, iscsi_name)) + self._execute("sudo ietadm --op new --tid=%s " + "--lun=0 --params Path=%s,Type=fileio" % + (iscsi_target, volume_path)) + def remove_export(self, context, volume): """Removes an export for a logical volume.""" iscsi_target = self.db.volume_get_iscsi_target_num(context, volume['id']) - yield self._execute("sudo ietadm --op delete --tid=%s " - "--lun=0" % iscsi_target) - yield self._execute("sudo ietadm --op delete --tid=%s" % - iscsi_target) + self._execute("sudo ietadm --op delete --tid=%s " + "--lun=0" % iscsi_target) + self._execute("sudo ietadm --op delete --tid=%s" % + iscsi_target) - @defer.inlineCallbacks def _get_name_and_portal(self, volume_name, host): """Gets iscsi name and portal from volume name and host.""" - (out, _err) = yield self._execute("sudo iscsiadm -m discovery -t " - "sendtargets -p %s" % host) + (out, _err) = self._execute("sudo iscsiadm -m discovery -t " + "sendtargets -p %s" % host) for target in out.splitlines(): if FLAGS.iscsi_ip_prefix in target and volume_name in target: (location, _sep, iscsi_name) = target.partition(" ") break iscsi_portal = location.split(",")[0] - defer.returnValue((iscsi_name, iscsi_portal)) + return (iscsi_name, iscsi_portal) - @defer.inlineCallbacks def discover_volume(self, volume): """Discover volume on a remote host.""" - (iscsi_name, - iscsi_portal) = yield self._get_name_and_portal(volume['name'], - volume['host']) - yield self._execute("sudo iscsiadm -m node -T %s -p %s --login" % - (iscsi_name, iscsi_portal)) - yield self._execute("sudo iscsiadm -m node -T %s -p %s --op update " - "-n node.startup -v automatic" % - (iscsi_name, iscsi_portal)) - defer.returnValue("/dev/iscsi/%s" % volume['name']) - - @defer.inlineCallbacks + iscsi_name, iscsi_portal = self._get_name_and_portal(volume['name'], + volume['host']) + self._execute("sudo iscsiadm -m node -T %s -p %s --login" % + (iscsi_name, iscsi_portal)) + self._execute("sudo iscsiadm -m node -T %s -p %s --op update " + "-n node.startup -v automatic" % + (iscsi_name, iscsi_portal)) + return "/dev/iscsi/%s" % volume['name'] + def undiscover_volume(self, volume): """Undiscover volume on a remote host.""" - (iscsi_name, - iscsi_portal) = yield self._get_name_and_portal(volume['name'], - volume['host']) - yield self._execute("sudo iscsiadm -m node -T %s -p %s --op update " - "-n node.startup -v manual" % - (iscsi_name, iscsi_portal)) - yield self._execute("sudo iscsiadm -m node -T %s -p %s --logout " % - (iscsi_name, iscsi_portal)) - yield self._execute("sudo iscsiadm -m node --op delete " - "--targetname %s" % iscsi_name) + iscsi_name, iscsi_portal = self._get_name_and_portal(volume['name'], + volume['host']) + self._execute("sudo iscsiadm -m node -T %s -p %s --op update " + "-n node.startup -v manual" % + (iscsi_name, iscsi_portal)) + self._execute("sudo iscsiadm -m node -T %s -p %s --logout " % + (iscsi_name, iscsi_portal)) + self._execute("sudo iscsiadm -m node --op delete " + "--targetname %s" % iscsi_name) class FakeISCSIDriver(ISCSIDriver): @@ -332,5 +310,5 @@ class FakeISCSIDriver(ISCSIDriver): @staticmethod def fake_execute(cmd, *_args, **_kwargs): """Execute that simply logs the command.""" - logging.debug("FAKE ISCSI: %s", cmd) + logging.debug(_("FAKE ISCSI: %s"), cmd) return (None, None) diff --git a/nova/volume/manager.py b/nova/volume/manager.py index 589e7d7d9..966334c50 100644 --- a/nova/volume/manager.py +++ b/nova/volume/manager.py @@ -45,7 +45,6 @@ intact. import logging import datetime -from twisted.internet import defer from nova import context from nova import exception @@ -82,16 +81,15 @@ class VolumeManager(manager.Manager): self.driver.check_for_setup_error() ctxt = context.get_admin_context() volumes = self.db.volume_get_all_by_host(ctxt, self.host) - logging.debug("Re-exporting %s volumes", len(volumes)) + logging.debug(_("Re-exporting %s volumes"), len(volumes)) for volume in volumes: self.driver.ensure_export(ctxt, volume) - @defer.inlineCallbacks def create_volume(self, context, volume_id): """Creates and exports the volume.""" context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) - logging.info("volume %s: creating", volume_ref['name']) + logging.info(_("volume %s: creating"), volume_ref['name']) self.db.volume_update(context, volume_id, @@ -100,38 +98,36 @@ class VolumeManager(manager.Manager): # before passing it to the driver. volume_ref['host'] = self.host - logging.debug("volume %s: creating lv of size %sG", + logging.debug(_("volume %s: creating lv of size %sG"), volume_ref['name'], volume_ref['size']) - yield self.driver.create_volume(volume_ref) + self.driver.create_volume(volume_ref) - logging.debug("volume %s: creating export", volume_ref['name']) - yield self.driver.create_export(context, volume_ref) + logging.debug(_("volume %s: creating export"), volume_ref['name']) + self.driver.create_export(context, volume_ref) now = datetime.datetime.utcnow() self.db.volume_update(context, volume_ref['id'], {'status': 'available', 'launched_at': now}) - logging.debug("volume %s: created successfully", volume_ref['name']) - defer.returnValue(volume_id) + logging.debug(_("volume %s: created successfully"), volume_ref['name']) + return volume_id - @defer.inlineCallbacks def delete_volume(self, context, volume_id): """Deletes and unexports volume.""" context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) if volume_ref['attach_status'] == "attached": - raise exception.Error("Volume is still attached") + raise exception.Error(_("Volume is still attached")) if volume_ref['host'] != self.host: - raise exception.Error("Volume is not local to this node") - logging.debug("volume %s: removing export", volume_ref['name']) - yield self.driver.remove_export(context, volume_ref) - logging.debug("volume %s: deleting", volume_ref['name']) - yield self.driver.delete_volume(volume_ref) + raise exception.Error(_("Volume is not local to this node")) + logging.debug(_("volume %s: removing export"), volume_ref['name']) + self.driver.remove_export(context, volume_ref) + logging.debug(_("volume %s: deleting"), volume_ref['name']) + self.driver.delete_volume(volume_ref) self.db.volume_destroy(context, volume_id) - logging.debug("volume %s: deleted successfully", volume_ref['name']) - defer.returnValue(True) + logging.debug(_("volume %s: deleted successfully"), volume_ref['name']) + return True - @defer.inlineCallbacks def setup_compute_volume(self, context, volume_id): """Setup remote volume on compute host. @@ -139,17 +135,16 @@ class VolumeManager(manager.Manager): context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) if volume_ref['host'] == self.host and FLAGS.use_local_volumes: - path = yield self.driver.local_path(volume_ref) + path = self.driver.local_path(volume_ref) else: - path = yield self.driver.discover_volume(volume_ref) - defer.returnValue(path) + path = self.driver.discover_volume(volume_ref) + return path - @defer.inlineCallbacks def remove_compute_volume(self, context, volume_id): """Remove remote volume on compute host.""" context = context.elevated() volume_ref = self.db.volume_get(context, volume_id) if volume_ref['host'] == self.host and FLAGS.use_local_volumes: - defer.returnValue(True) + return True else: - yield self.driver.undiscover_volume(volume_ref) + self.driver.undiscover_volume(volume_ref) diff --git a/run_tests.py b/run_tests.py deleted file mode 100644 index 3d427d8af..000000000 --- a/run_tests.py +++ /dev/null @@ -1,122 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -This is our basic test running framework based on Twisted's Trial. - -Usage Examples: - - # to run all the tests - python run_tests.py - - # to run a specific test suite imported here - python run_tests.py NodeConnectionTestCase - - # to run a specific test imported here - python run_tests.py NodeConnectionTestCase.test_reboot - - # to run some test suites elsewhere - python run_tests.py nova.tests.node_unittest - python run_tests.py nova.tests.node_unittest.NodeConnectionTestCase - -Due to our use of multiprocessing it we frequently get some ignorable -'Interrupted system call' exceptions after test completion. - -""" - -import __main__ -import os -import sys - -from twisted.scripts import trial as trial_script - -from nova import flags -from nova import twistd - -from nova.tests.access_unittest import * -from nova.tests.api_unittest import * -from nova.tests.auth_unittest import * -from nova.tests.cloud_unittest import * -from nova.tests.compute_unittest import * -from nova.tests.flags_unittest import * -from nova.tests.misc_unittest import * -from nova.tests.network_unittest import * -from nova.tests.objectstore_unittest import * -from nova.tests.process_unittest import * -from nova.tests.quota_unittest import * -from nova.tests.rpc_unittest import * -from nova.tests.scheduler_unittest import * -from nova.tests.service_unittest import * -from nova.tests.twistd_unittest import * -from nova.tests.validator_unittest import * -from nova.tests.virt_unittest import * -from nova.tests.virt_unittest import * -from nova.tests.volume_unittest import * - - -FLAGS = flags.FLAGS -flags.DEFINE_bool('flush_db', True, - 'Flush the database before running fake tests') -flags.DEFINE_string('tests_stderr', 'run_tests.err.log', - 'Path to where to pipe STDERR during test runs.' - ' Default = "run_tests.err.log"') - - -if __name__ == '__main__': - OptionsClass = twistd.WrapTwistedOptions(trial_script.Options) - config = OptionsClass() - argv = config.parseOptions() - - FLAGS.verbose = True - - # TODO(termie): these should make a call instead of doing work on import - if FLAGS.fake_tests: - from nova.tests.fake_flags import * - else: - from nova.tests.real_flags import * - - # Establish redirect for STDERR - sys.stderr.flush() - err = open(FLAGS.tests_stderr, 'w+', 0) - os.dup2(err.fileno(), sys.stderr.fileno()) - - if len(argv) == 1 and len(config['tests']) == 0: - # If no tests were specified run the ones imported in this file - # NOTE(termie): "tests" is not a flag, just some Trial related stuff - config['tests'].update(['__main__']) - elif len(config['tests']): - # If we specified tests check first whether they are in __main__ - for arg in config['tests']: - key = arg.split('.')[0] - if hasattr(__main__, key): - config['tests'].remove(arg) - config['tests'].add('__main__.%s' % arg) - - trial_script._initialDebugSetup(config) - trialRunner = trial_script._makeRunner(config) - suite = trial_script._getSuite(config) - if config['until-failure']: - test_result = trialRunner.runUntilFailure(suite) - else: - test_result = trialRunner.run(suite) - if config.tracer: - sys.settrace(None) - results = config.tracer.results() - results.write_results(show_missing=1, summary=False, - coverdir=config.coverdir) - sys.exit(not test_result.wasSuccessful()) diff --git a/run_tests.sh b/run_tests.sh index a11dcd7cc..67214996d 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -36,7 +36,8 @@ done if [ $never_venv -eq 1 ]; then # Just run the test suites in current environment - python run_tests.py + rm -f nova.sqlite + nosetests -v exit fi @@ -47,7 +48,8 @@ if [ $force -eq 1 ]; then fi if [ -e ${venv} ]; then - ${with_venv} python run_tests.py $@ + ${with_venv} rm -f nova.sqlite + ${with_venv} nosetests -v $@ else if [ $always_venv -eq 1 ]; then # Automatically install the virtualenv @@ -59,9 +61,11 @@ else # Install the virtualenv and run the test suite in it python tools/install_venv.py else - python run_tests.py + rm -f nova.sqlite + nosetests -v exit fi fi - ${with_venv} python run_tests.py $@ + ${with_venv} rm -f nova.sqlite + ${with_venv} nosetests -v $@ fi @@ -59,6 +59,7 @@ setup(name='nova', 'build_sphinx': local_BuildDoc}, packages=find_packages(exclude=['bin', 'smoketests']), include_package_data=True, + test_suite='nose.collector', scripts=['bin/nova-api', 'bin/nova-compute', 'bin/nova-dhcpbridge', diff --git a/smoketests/admin_smoketests.py b/smoketests/admin_smoketests.py new file mode 100644 index 000000000..50bb3fa2e --- /dev/null +++ b/smoketests/admin_smoketests.py @@ -0,0 +1,92 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import random +import sys +import time +import unittest +import zipfile + +from nova import adminclient +from smoketests import flags +from smoketests import base + + +SUITE_NAMES = '[user]' + +FLAGS = flags.FLAGS +flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) + +# TODO(devamcar): Use random tempfile +ZIP_FILENAME = '/tmp/nova-me-x509.zip' + +TEST_PREFIX = 'test%s' % int(random.random()*1000000) +TEST_USERNAME = '%suser' % TEST_PREFIX +TEST_PROJECTNAME = '%sproject' % TEST_PREFIX + + +class AdminSmokeTestCase(base.SmokeTestCase): + def setUp(self): + self.admin = adminclient.NovaAdminClient( + access_key=os.getenv('EC2_ACCESS_KEY'), + secret_key=os.getenv('EC2_SECRET_KEY'), + clc_url=os.getenv('EC2_URL'), + region=FLAGS.region) + + +class UserTests(AdminSmokeTestCase): + """ Test admin credentials and user creation. """ + + def test_001_admin_can_connect(self): + conn = self.admin.connection_for('admin', 'admin') + self.assert_(conn) + + def test_002_admin_can_create_user(self): + user = self.admin.create_user(TEST_USERNAME) + self.assertEqual(user.username, TEST_USERNAME) + + def test_003_admin_can_create_project(self): + project = self.admin.create_project(TEST_PROJECTNAME, + TEST_USERNAME) + self.assertEqual(project.projectname, TEST_PROJECTNAME) + + def test_004_user_can_download_credentials(self): + buf = self.admin.get_zip(TEST_USERNAME, TEST_PROJECTNAME) + output = open(ZIP_FILENAME, 'w') + output.write(buf) + output.close() + + zip = zipfile.ZipFile(ZIP_FILENAME, 'a', zipfile.ZIP_DEFLATED) + bad = zip.testzip() + zip.close() + + self.failIf(bad) + + def test_999_tearDown(self): + self.admin.delete_project(TEST_PROJECTNAME) + self.admin.delete_user(TEST_USERNAME) + try: + os.remove(ZIP_FILENAME) + except: + pass + +if __name__ == "__main__": + suites = {'user': unittest.makeSuite(UserTests)} + sys.exit(base.run_tests(suites)) + diff --git a/smoketests/novatestcase.py b/smoketests/base.py index 513e0ca91..5a14d3e09 100644 --- a/smoketests/novatestcase.py +++ b/smoketests/base.py @@ -16,36 +16,26 @@ # License for the specific language governing permissions and limitations # under the License. +import boto import commands +import httplib import os +import paramiko import random import sys import unittest +from boto.ec2.regioninfo import RegionInfo - -import paramiko - -from nova import adminclient from smoketests import flags FLAGS = flags.FLAGS -class NovaTestCase(unittest.TestCase): - def setUp(self): - self.nova_admin = adminclient.NovaAdminClient( - access_key=FLAGS.admin_access_key, - secret_key=FLAGS.admin_secret_key, - clc_ip=FLAGS.clc_ip) - - def tearDown(self): - pass - +class SmokeTestCase(unittest.TestCase): def connect_ssh(self, ip, key_name): # TODO(devcamcar): set a more reasonable connection timeout time key = paramiko.RSAKey.from_private_key_file('/tmp/%s.pem' % key_name) client = paramiko.SSHClient() - client.load_system_host_keys() client.set_missing_host_key_policy(paramiko.WarningPolicy()) client.connect(ip, username='root', pkey=key) stdin, stdout, stderr = client.exec_command('uptime') @@ -53,26 +43,50 @@ class NovaTestCase(unittest.TestCase): return client def can_ping(self, ip): - return commands.getstatusoutput('ping -c 1 %s' % ip)[0] == 0 - - @property - def admin(self): - return self.nova_admin.connection_for('admin') - - def connection_for(self, username): - return self.nova_admin.connection_for(username) - - def create_user(self, username): - return self.nova_admin.create_user(username) - - def get_user(self, username): - return self.nova_admin.get_user(username) - - def delete_user(self, username): - return self.nova_admin.delete_user(username) - - def get_signed_zip(self, username): - return self.nova_admin.get_zip(username) + """ Attempt to ping the specified IP, and give up after 1 second. """ + + # NOTE(devcamcar): ping timeout flag is different in OSX. + if sys.platform == 'darwin': + timeout_flag = 't' + else: + timeout_flag = 'w' + + status, output = commands.getstatusoutput('ping -c1 -%s1 %s' % + (timeout_flag, ip)) + return status == 0 + + def connection_for_env(self, **kwargs): + """ + Returns a boto ec2 connection for the current environment. + """ + access_key = os.getenv('EC2_ACCESS_KEY') + secret_key = os.getenv('EC2_SECRET_KEY') + clc_url = os.getenv('EC2_URL') + + if not access_key or not secret_key or not clc_url: + raise Exception('Missing EC2 environment variables. Please source ' + 'the appropriate novarc file before running this ' + 'test.') + + parts = self.split_clc_url(clc_url) + return boto.connect_ec2(aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + is_secure=parts['is_secure'], + region=RegionInfo(None, + 'nova', + parts['ip']), + port=parts['port'], + path='/services/Cloud', + **kwargs) + + def split_clc_url(self, clc_url): + """ + Splits a cloud controller endpoint url. + """ + parts = httplib.urlsplit(clc_url) + is_secure = parts.scheme == 'https' + ip, port = parts.netloc.split(':') + return {'ip': ip, 'port': int(port), 'is_secure': is_secure} def create_key_pair(self, conn, key_name): try: @@ -116,15 +130,25 @@ class NovaTestCase(unittest.TestCase): raise Exception(output) return True - def register_image(self, bucket_name, manifest): - conn = nova_admin.connection_for('admin') - return conn.register_image("%s/%s.manifest.xml" % (bucket_name, manifest)) +def run_tests(suites): + argv = FLAGS(sys.argv) + + if not os.getenv('EC2_ACCESS_KEY'): + print >> sys.stderr, 'Missing EC2 environment variables. Please ' \ + 'source the appropriate novarc file before ' \ + 'running this test.' + return 1 + + if FLAGS.suite: + try: + suite = suites[FLAGS.suite] + except KeyError: + print >> sys.stderr, 'Available test suites:', \ + ', '.join(suites.keys()) + return 1 - def setUp_test_image(self, image, kernel=False): - self.bundle_image(image, kernel=kernel) - bucket = "auto_test_%s" % int(random.random() * 1000000) - self.upload_image(bucket, image) - return self.register_image(bucket, image) + unittest.TextTestRunner(verbosity=2).run(suite) + else: + for suite in suites.itervalues(): + unittest.TextTestRunner(verbosity=2).run(suite) - def tearDown_test_image(self, conn, image_id): - conn.deregister_image(image_id) diff --git a/smoketests/flags.py b/smoketests/flags.py index 3617fb797..ae4d09508 100644 --- a/smoketests/flags.py +++ b/smoketests/flags.py @@ -1,7 +1,7 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. +# Administrator of the National Aeronautics and Space Administration. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -33,13 +33,6 @@ DEFINE_bool = DEFINE_bool # __GLOBAL FLAGS ONLY__ # Define any app-specific flags in their own files, docs at: # http://code.google.com/p/python-gflags/source/browse/trunk/gflags.py#39 -DEFINE_string('admin_access_key', 'admin', 'Access key for admin user') -DEFINE_string('admin_secret_key', 'admin', 'Secret key for admin user') -DEFINE_string('clc_ip', '127.0.0.1', 'IP of cloud controller API') -DEFINE_string('bundle_kernel', 'openwrt-x86-vmlinuz', - 'Local kernel file to use for bundling tests') -DEFINE_string('bundle_image', 'openwrt-x86-ext2.image', - 'Local image file to use for bundling tests') -#DEFINE_string('vpn_image_id', 'ami-CLOUDPIPE', -# 'AMI for cloudpipe vpn server') +DEFINE_string('region', 'nova', 'Region to use') +DEFINE_string('test_image', 'ami-tiny', 'Image to use for launch tests') diff --git a/smoketests/smoketest.py b/smoketests/smoketest.py deleted file mode 100644 index ad95114d4..000000000 --- a/smoketests/smoketest.py +++ /dev/null @@ -1,566 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import commands -import os -import random -import re -import sys -import time -import unittest -import zipfile - - -import paramiko - -from smoketests import flags -from smoketests import novatestcase - -SUITE_NAMES = '[user, image, security, public_network, volume]' - -FLAGS = flags.FLAGS -flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) - -# TODO(devamcar): Use random tempfile -ZIP_FILENAME = '/tmp/nova-me-x509.zip' - -data = {} - -test_prefix = 'test%s' % int(random.random()*1000000) -test_username = '%suser' % test_prefix -test_bucket = '%s_bucket' % test_prefix -test_key = '%s_key' % test_prefix - -# Test admin credentials and user creation -class UserTests(novatestcase.NovaTestCase): - def test_001_admin_can_connect(self): - conn = self.connection_for('admin') - self.assert_(conn) - - def test_002_admin_can_create_user(self): - userinfo = self.create_user(test_username) - self.assertEqual(userinfo.username, test_username) - - def test_003_user_can_download_credentials(self): - buf = self.get_signed_zip(test_username) - output = open(ZIP_FILENAME, 'w') - output.write(buf) - output.close() - - zip = zipfile.ZipFile(ZIP_FILENAME, 'a', zipfile.ZIP_DEFLATED) - bad = zip.testzip() - zip.close() - - self.failIf(bad) - - def test_999_tearDown(self): - self.delete_user(test_username) - user = self.get_user(test_username) - self.assert_(user is None) - try: - os.remove(ZIP_FILENAME) - except: - pass - -# Test image bundling, registration, and launching -class ImageTests(novatestcase.NovaTestCase): - def test_000_setUp(self): - self.create_user(test_username) - - def test_001_admin_can_bundle_image(self): - self.assertTrue(self.bundle_image(FLAGS.bundle_image)) - - def test_002_admin_can_upload_image(self): - self.assertTrue(self.upload_image(test_bucket, FLAGS.bundle_image)) - - def test_003_admin_can_register_image(self): - image_id = self.register_image(test_bucket, FLAGS.bundle_image) - self.assert_(image_id is not None) - data['image_id'] = image_id - - def test_004_admin_can_bundle_kernel(self): - self.assertTrue(self.bundle_image(FLAGS.bundle_kernel, kernel=True)) - - def test_005_admin_can_upload_kernel(self): - self.assertTrue(self.upload_image(test_bucket, FLAGS.bundle_kernel)) - - def test_006_admin_can_register_kernel(self): - # FIXME(devcamcar): registration should verify that bucket/manifest - # exists before returning successfully. - kernel_id = self.register_image(test_bucket, FLAGS.bundle_kernel) - self.assert_(kernel_id is not None) - data['kernel_id'] = kernel_id - - def test_007_admin_images_are_available_within_10_seconds(self): - for i in xrange(10): - image = self.admin.get_image(data['image_id']) - if image and image.state == 'available': - break - time.sleep(1) - else: - print image.state - self.assert_(False) # wasn't available within 10 seconds - self.assert_(image.type == 'machine') - - for i in xrange(10): - kernel = self.admin.get_image(data['kernel_id']) - if kernel and kernel.state == 'available': - break - time.sleep(1) - else: - self.assert_(False) # wasn't available within 10 seconds - self.assert_(kernel.type == 'kernel') - - def test_008_admin_can_describe_image_attribute(self): - attrs = self.admin.get_image_attribute(data['image_id'], - 'launchPermission') - self.assert_(attrs.name, 'launch_permission') - - def test_009_me_cannot_see_non_public_images(self): - conn = self.connection_for(test_username) - images = conn.get_all_images(image_ids=[data['image_id']]) - self.assertEqual(len(images), 0) - - def test_010_admin_can_modify_image_launch_permission(self): - conn = self.connection_for(test_username) - - self.admin.modify_image_attribute(image_id=data['image_id'], - operation='add', - attribute='launchPermission', - groups='all') - - image = conn.get_image(data['image_id']) - self.assertEqual(image.id, data['image_id']) - - def test_011_me_can_list_public_images(self): - conn = self.connection_for(test_username) - images = conn.get_all_images(image_ids=[data['image_id']]) - self.assertEqual(len(images), 1) - pass - - def test_012_me_can_see_launch_permission(self): - attrs = self.admin.get_image_attribute(data['image_id'], - 'launchPermission') - self.assert_(attrs.name, 'launch_permission') - self.assert_(attrs.groups[0], 'all') - - # FIXME: add tests that user can launch image - -# def test_013_user_can_launch_admin_public_image(self): -# # TODO: Use openwrt kernel instead of default kernel -# conn = self.connection_for(test_username) -# reservation = conn.run_instances(data['image_id']) -# self.assertEqual(len(reservation.instances), 1) -# data['my_instance_id'] = reservation.instances[0].id - -# def test_014_instances_launch_within_30_seconds(self): -# pass - -# def test_015_user_can_terminate(self): -# conn = self.connection_for(test_username) -# terminated = conn.terminate_instances( -# instance_ids=[data['my_instance_id']]) -# self.assertEqual(len(terminated), 1) - - def test_016_admin_can_deregister_kernel(self): - self.assertTrue(self.admin.deregister_image(data['kernel_id'])) - - def test_017_admin_can_deregister_image(self): - self.assertTrue(self.admin.deregister_image(data['image_id'])) - - def test_018_admin_can_delete_bundle(self): - self.assertTrue(self.delete_bundle_bucket(test_bucket)) - - def test_999_tearDown(self): - data = {} - self.delete_user(test_username) - - -# Test key pairs and security groups -class SecurityTests(novatestcase.NovaTestCase): - def test_000_setUp(self): - self.create_user(test_username + '_me') - self.create_user(test_username + '_you') - data['image_id'] = 'ami-tiny' - - def test_001_me_can_create_keypair(self): - conn = self.connection_for(test_username + '_me') - key = self.create_key_pair(conn, test_key) - self.assertEqual(key.name, test_key) - - def test_002_you_can_create_keypair(self): - conn = self.connection_for(test_username + '_you') - key = self.create_key_pair(conn, test_key+ 'yourkey') - self.assertEqual(key.name, test_key+'yourkey') - - def test_003_me_can_create_instance_with_keypair(self): - conn = self.connection_for(test_username + '_me') - reservation = conn.run_instances(data['image_id'], key_name=test_key) - self.assertEqual(len(reservation.instances), 1) - data['my_instance_id'] = reservation.instances[0].id - - def test_004_me_can_obtain_private_ip_within_60_seconds(self): - conn = self.connection_for(test_username + '_me') - reservations = conn.get_all_instances([data['my_instance_id']]) - instance = reservations[0].instances[0] - # allow 60 seconds to exit pending with IP - for x in xrange(60): - instance.update() - if instance.state != u'pending': - break - time.sleep(1) - else: - self.assert_(False) - # self.assertEqual(instance.state, u'running') - ip = reservations[0].instances[0].private_dns_name - self.failIf(ip == '0.0.0.0') - data['my_private_ip'] = ip - print data['my_private_ip'], - - def test_005_can_ping_private_ip(self): - for x in xrange(120): - # ping waits for 1 second - status, output = commands.getstatusoutput( - 'ping -c1 -w1 %s' % data['my_private_ip']) - if status == 0: - break - else: - self.assert_('could not ping instance') - #def test_005_me_cannot_ssh_when_unauthorized(self): - # self.assertRaises(paramiko.SSHException, self.connect_ssh, - # data['my_private_ip'], 'mykey') - - #def test_006_me_can_authorize_ssh(self): - # conn = self.connection_for(test_username + '_me') - # self.assertTrue( - # conn.authorize_security_group( - # 'default', - # ip_protocol='tcp', - # from_port=22, - # to_port=22, - # cidr_ip='0.0.0.0/0' - # ) - # ) - - def test_007_me_can_ssh_when_authorized(self): - conn = self.connect_ssh(data['my_private_ip'], test_key) - conn.close() - - #def test_008_me_can_revoke_ssh_authorization(self): - # conn = self.connection_for('me') - # self.assertTrue( - # conn.revoke_security_group( - # 'default', - # ip_protocol='tcp', - # from_port=22, - # to_port=22, - # cidr_ip='0.0.0.0/0' - # ) - # ) - - #def test_009_you_cannot_ping_my_instance(self): - # TODO: should ping my_private_ip from with an instance started by you. - #self.assertFalse(self.can_ping(data['my_private_ip'])) - - def test_010_you_cannot_ssh_to_my_instance(self): - try: - conn = self.connect_ssh(data['my_private_ip'], - test_key + 'yourkey') - conn.close() - except paramiko.SSHException: - pass - else: - self.fail("expected SSHException") - - def test_999_tearDown(self): - conn = self.connection_for(test_username + '_me') - self.delete_key_pair(conn, test_key) - if data.has_key('my_instance_id'): - conn.terminate_instances([data['my_instance_id']]) - - conn = self.connection_for(test_username + '_you') - self.delete_key_pair(conn, test_key + 'yourkey') - - conn = self.connection_for('admin') - self.delete_user(test_username + '_me') - self.delete_user(test_username + '_you') - #self.tearDown_test_image(conn, data['image_id']) - -# TODO: verify wrt image boots -# build python into wrt image -# build boto/m2crypto into wrt image -# build euca2ools into wrt image -# build a script to download and unpack credentials -# - return "ok" to stdout for comparison in self.assertEqual() -# build a script to bundle the instance -# build a script to upload the bundle - -# status, output = commands.getstatusoutput('cmd') -# if status == 0: -# print 'ok' -# else: -# print output - -# Testing rebundling -class RebundlingTests(novatestcase.NovaTestCase): - def test_000_setUp(self): - self.create_user('me') - self.create_user('you') - # TODO: create keypair for me - # upload smoketest img - # run instance - - def test_001_me_can_download_credentials_within_instance(self): - conn = self.connect_ssh(data['my_private_ip'], 'mykey') - stdin, stdout = conn.exec_command( - 'python ~/smoketests/install-credentials.py') - conn.close() - self.assertEqual(stdout, 'ok') - - def test_002_me_can_rebundle_within_instance(self): - conn = self.connect_ssh(data['my_private_ip'], 'mykey') - stdin, stdout = conn.exec_command( - 'python ~/smoketests/rebundle-instance.py') - conn.close() - self.assertEqual(stdout, 'ok') - - def test_003_me_can_upload_image_within_instance(self): - conn = self.connect_ssh(data['my_private_ip'], 'mykey') - stdin, stdout = conn.exec_command( - 'python ~/smoketests/upload-bundle.py') - conn.close() - self.assertEqual(stdout, 'ok') - - def test_004_me_can_register_image_within_instance(self): - conn = self.connect_ssh(data['my_private_ip'], 'mykey') - stdin, stdout = conn.exec_command( - 'python ~/smoketests/register-image.py') - conn.close() - if re.matches('ami-{\w+}', stdout): - data['my_image_id'] = stdout.strip() - else: - self.fail('expected ami-nnnnnn, got:\n ' + stdout) - - def test_005_you_cannot_see_my_private_image(self): - conn = self.connection_for('you') - image = conn.get_image(data['my_image_id']) - self.assertEqual(image, None) - - def test_006_me_can_make_image_public(self): - conn = self.connection_for(test_username) - conn.modify_image_attribute(image_id=data['my_image_id'], - operation='add', - attribute='launchPermission', - groups='all') - - def test_007_you_can_see_my_public_image(self): - conn = self.connection_for('you') - image = conn.get_image(data['my_image_id']) - self.assertEqual(image.id, data['my_image_id']) - - def test_999_tearDown(self): - self.delete_user('me') - self.delete_user('you') - - #if data.has_key('image_id'): - # deregister rebundled image - - # TODO: tear down instance - # delete keypairs - data = {} - -# Test elastic IPs -class ElasticIPTests(novatestcase.NovaTestCase): - def test_000_setUp(self): - data['image_id'] = 'ami-tiny' - - self.create_user('me') - conn = self.connection_for('me') - self.create_key_pair(conn, 'mykey') - - conn = self.connection_for('admin') - #data['image_id'] = self.setUp_test_image(FLAGS.bundle_image) - - def test_001_me_can_launch_image_with_keypair(self): - conn = self.connection_for('me') - reservation = conn.run_instances(data['image_id'], key_name='mykey') - self.assertEqual(len(reservation.instances), 1) - data['my_instance_id'] = reservation.instances[0].id - - def test_002_me_can_allocate_elastic_ip(self): - conn = self.connection_for('me') - data['my_public_ip'] = conn.allocate_address() - self.assert_(data['my_public_ip'].public_ip) - - def test_003_me_can_associate_ip_with_instance(self): - self.assertTrue(data['my_public_ip'].associate(data['my_instance_id'])) - - def test_004_me_can_ssh_with_public_ip(self): - conn = self.connect_ssh(data['my_public_ip'].public_ip, 'mykey') - conn.close() - - def test_005_me_can_disassociate_ip_from_instance(self): - self.assertTrue(data['my_public_ip'].disassociate()) - - def test_006_me_can_deallocate_elastic_ip(self): - self.assertTrue(data['my_public_ip'].delete()) - - def test_999_tearDown(self): - conn = self.connection_for('me') - self.delete_key_pair(conn, 'mykey') - - conn = self.connection_for('admin') - #self.tearDown_test_image(conn, data['image_id']) - data = {} - -ZONE = 'nova' -DEVICE = 'vdb' -# Test iscsi volumes -class VolumeTests(novatestcase.NovaTestCase): - def test_000_setUp(self): - self.create_user(test_username) - data['image_id'] = 'ami-tiny' # A7370FE3 - - conn = self.connection_for(test_username) - self.create_key_pair(conn, test_key) - reservation = conn.run_instances(data['image_id'], - instance_type='m1.tiny', - key_name=test_key) - data['instance_id'] = reservation.instances[0].id - data['private_ip'] = reservation.instances[0].private_dns_name - # wait for instance to show up - for x in xrange(120): - # ping waits for 1 second - status, output = commands.getstatusoutput( - 'ping -c1 -w1 %s' % data['private_ip']) - if status == 0: - break - else: - self.fail('unable to ping instance') - - def test_001_me_can_create_volume(self): - conn = self.connection_for(test_username) - volume = conn.create_volume(1, ZONE) - self.assertEqual(volume.size, 1) - data['volume_id'] = volume.id - # give network time to find volume - time.sleep(5) - - def test_002_me_can_attach_volume(self): - conn = self.connection_for(test_username) - conn.attach_volume( - volume_id = data['volume_id'], - instance_id = data['instance_id'], - device = '/dev/%s' % DEVICE - ) - # give instance time to recognize volume - time.sleep(5) - - def test_003_me_can_mount_volume(self): - conn = self.connect_ssh(data['private_ip'], test_key) - # FIXME(devcamcar): the tiny image doesn't create the node properly - # this will make /dev/vd* if it doesn't exist - stdin, stdout, stderr = conn.exec_command( - 'grep %s /proc/partitions |' + \ - '`awk \'{print "mknod /dev/"$4" b "$1" "$2}\'`' % DEVICE) - commands = [] - commands.append('mkdir -p /mnt/vol') - commands.append('mkfs.ext2 /dev/%s' % DEVICE) - commands.append('mount /dev/%s /mnt/vol' % DEVICE) - commands.append('echo success') - stdin, stdout, stderr = conn.exec_command(' && '.join(commands)) - out = stdout.read() - conn.close() - if not out.strip().endswith('success'): - self.fail('Unable to mount: %s %s' % (out, stderr.read())) - - def test_004_me_can_write_to_volume(self): - conn = self.connect_ssh(data['private_ip'], test_key) - # FIXME(devcamcar): This doesn't fail if the volume hasn't been mounted - stdin, stdout, stderr = conn.exec_command( - 'echo hello > /mnt/vol/test.txt') - err = stderr.read() - conn.close() - if len(err) > 0: - self.fail('Unable to write to mount: %s' % (err)) - - def test_005_volume_is_correct_size(self): - conn = self.connect_ssh(data['private_ip'], test_key) - stdin, stdout, stderr = conn.exec_command( - "df -h | grep %s | awk {'print $2'}" % DEVICE) - out = stdout.read() - conn.close() - if not out.strip() == '1007.9M': - self.fail('Volume is not the right size: %s %s' % (out, stderr.read())) - - def test_006_me_can_umount_volume(self): - conn = self.connect_ssh(data['private_ip'], test_key) - stdin, stdout, stderr = conn.exec_command('umount /mnt/vol') - err = stderr.read() - conn.close() - if len(err) > 0: - self.fail('Unable to unmount: %s' % (err)) - - def test_007_me_can_detach_volume(self): - conn = self.connection_for(test_username) - self.assertTrue(conn.detach_volume(volume_id = data['volume_id'])) - - def test_008_me_can_delete_volume(self): - conn = self.connection_for(test_username) - self.assertTrue(conn.delete_volume(data['volume_id'])) - - def test_009_volume_size_must_be_int(self): - conn = self.connection_for(test_username) - self.assertRaises(Exception, conn.create_volume, 'foo', ZONE) - - def test_999_tearDown(self): - global data - conn = self.connection_for(test_username) - self.delete_key_pair(conn, test_key) - if data.has_key('instance_id'): - conn.terminate_instances([data['instance_id']]) - self.delete_user(test_username) - data = {} - -def build_suites(): - return { - 'user': unittest.makeSuite(UserTests), - 'image': unittest.makeSuite(ImageTests), - 'security': unittest.makeSuite(SecurityTests), - 'public_network': unittest.makeSuite(ElasticIPTests), - 'volume': unittest.makeSuite(VolumeTests), - } - -def main(): - argv = FLAGS(sys.argv) - suites = build_suites() - - if FLAGS.suite: - try: - suite = suites[FLAGS.suite] - except KeyError: - print >> sys.stderr, 'Available test suites:', SUITE_NAMES - return 1 - - unittest.TextTestRunner(verbosity=2).run(suite) - else: - for suite in suites.itervalues(): - unittest.TextTestRunner(verbosity=2).run(suite) - -if __name__ == "__main__": - sys.exit(main()) diff --git a/smoketests/user_smoketests.py b/smoketests/user_smoketests.py new file mode 100644 index 000000000..d29e3aea3 --- /dev/null +++ b/smoketests/user_smoketests.py @@ -0,0 +1,326 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import commands +import os +import random +import socket +import sys +import time +import unittest + +from smoketests import flags +from smoketests import base + + +SUITE_NAMES = '[image, instance, volume]' + +FLAGS = flags.FLAGS +flags.DEFINE_string('suite', None, 'Specific test suite to run ' + SUITE_NAMES) +flags.DEFINE_string('bundle_kernel', 'openwrt-x86-vmlinuz', + 'Local kernel file to use for bundling tests') +flags.DEFINE_string('bundle_image', 'openwrt-x86-ext2.image', + 'Local image file to use for bundling tests') + +TEST_PREFIX = 'test%s' % int (random.random()*1000000) +TEST_BUCKET = '%s_bucket' % TEST_PREFIX +TEST_KEY = '%s_key' % TEST_PREFIX +TEST_DATA = {} + + +class UserSmokeTestCase(base.SmokeTestCase): + def setUp(self): + global TEST_DATA + self.conn = self.connection_for_env() + self.data = TEST_DATA + + +class ImageTests(UserSmokeTestCase): + def test_001_can_bundle_image(self): + self.assertTrue(self.bundle_image(FLAGS.bundle_image)) + + def test_002_can_upload_image(self): + self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_image)) + + def test_003_can_register_image(self): + image_id = self.conn.register_image('%s/%s.manifest.xml' % + (TEST_BUCKET, FLAGS.bundle_image)) + self.assert_(image_id is not None) + self.data['image_id'] = image_id + + def test_004_can_bundle_kernel(self): + self.assertTrue(self.bundle_image(FLAGS.bundle_kernel, kernel=True)) + + def test_005_can_upload_kernel(self): + self.assertTrue(self.upload_image(TEST_BUCKET, FLAGS.bundle_kernel)) + + def test_006_can_register_kernel(self): + kernel_id = self.conn.register_image('%s/%s.manifest.xml' % + (TEST_BUCKET, FLAGS.bundle_kernel)) + self.assert_(kernel_id is not None) + self.data['kernel_id'] = kernel_id + + def test_007_images_are_available_within_10_seconds(self): + for i in xrange(10): + image = self.conn.get_image(self.data['image_id']) + if image and image.state == 'available': + break + time.sleep(1) + else: + print image.state + self.assert_(False) # wasn't available within 10 seconds + self.assert_(image.type == 'machine') + + for i in xrange(10): + kernel = self.conn.get_image(self.data['kernel_id']) + if kernel and kernel.state == 'available': + break + time.sleep(1) + else: + self.assert_(False) # wasn't available within 10 seconds + self.assert_(kernel.type == 'kernel') + + def test_008_can_describe_image_attribute(self): + attrs = self.conn.get_image_attribute(self.data['image_id'], + 'launchPermission') + self.assert_(attrs.name, 'launch_permission') + + def test_009_can_modify_image_launch_permission(self): + self.conn.modify_image_attribute(image_id=self.data['image_id'], + operation='add', + attribute='launchPermission', + groups='all') + image = self.conn.get_image(self.data['image_id']) + self.assertEqual(image.id, self.data['image_id']) + + def test_010_can_see_launch_permission(self): + attrs = self.conn.get_image_attribute(self.data['image_id'], + 'launchPermission') + self.assert_(attrs.name, 'launch_permission') + self.assert_(attrs.attrs['groups'][0], 'all') + + def test_011_user_can_deregister_kernel(self): + self.assertTrue(self.conn.deregister_image(self.data['kernel_id'])) + + def test_012_can_deregister_image(self): + self.assertTrue(self.conn.deregister_image(self.data['image_id'])) + + def test_013_can_delete_bundle(self): + self.assertTrue(self.delete_bundle_bucket(TEST_BUCKET)) + + +class InstanceTests(UserSmokeTestCase): + def test_001_can_create_keypair(self): + key = self.create_key_pair(self.conn, TEST_KEY) + self.assertEqual(key.name, TEST_KEY) + + def test_002_can_create_instance_with_keypair(self): + reservation = self.conn.run_instances(FLAGS.test_image, + key_name=TEST_KEY, + instance_type='m1.tiny') + self.assertEqual(len(reservation.instances), 1) + self.data['instance_id'] = reservation.instances[0].id + + def test_003_instance_runs_within_60_seconds(self): + reservations = self.conn.get_all_instances([data['instance_id']]) + instance = reservations[0].instances[0] + # allow 60 seconds to exit pending with IP + for x in xrange(60): + instance.update() + if instance.state == u'running': + break + time.sleep(1) + else: + self.fail('instance failed to start') + ip = reservations[0].instances[0].private_dns_name + self.failIf(ip == '0.0.0.0') + self.data['private_ip'] = ip + print self.data['private_ip'] + + def test_004_can_ping_private_ip(self): + for x in xrange(120): + # ping waits for 1 second + status, output = commands.getstatusoutput( + 'ping -c1 %s' % self.data['private_ip']) + if status == 0: + break + else: + self.fail('could not ping instance') + + def test_005_can_ssh_to_private_ip(self): + for x in xrange(30): + try: + conn = self.connect_ssh(self.data['private_ip'], TEST_KEY) + conn.close() + except Exception: + time.sleep(1) + else: + break + else: + self.fail('could not ssh to instance') + + def test_006_can_allocate_elastic_ip(self): + result = self.conn.allocate_address() + self.assertTrue(hasattr(result, 'public_ip')) + self.data['public_ip'] = result.public_ip + + def test_007_can_associate_ip_with_instance(self): + result = self.conn.associate_address(self.data['instance_id'], + self.data['public_ip']) + self.assertTrue(result) + + def test_008_can_ssh_with_public_ip(self): + for x in xrange(30): + try: + conn = self.connect_ssh(self.data['public_ip'], TEST_KEY) + conn.close() + except socket.error: + time.sleep(1) + else: + break + else: + self.fail('could not ssh to instance') + + def test_009_can_disassociate_ip_from_instance(self): + result = self.conn.disassociate_address(self.data['public_ip']) + self.assertTrue(result) + + def test_010_can_deallocate_elastic_ip(self): + result = self.conn.release_address(self.data['public_ip']) + self.assertTrue(result) + + def test_999_tearDown(self): + self.delete_key_pair(self.conn, TEST_KEY) + if self.data.has_key('instance_id'): + self.conn.terminate_instances([data['instance_id']]) + + +class VolumeTests(UserSmokeTestCase): + def setUp(self): + super(VolumeTests, self).setUp() + self.device = '/dev/vdb' + + def test_000_setUp(self): + self.create_key_pair(self.conn, TEST_KEY) + reservation = self.conn.run_instances(FLAGS.test_image, + instance_type='m1.tiny', + key_name=TEST_KEY) + instance = reservation.instances[0] + self.data['instance'] = instance + for x in xrange(120): + if self.can_ping(instance.private_dns_name): + break + else: + self.fail('unable to start instance') + + def test_001_can_create_volume(self): + volume = self.conn.create_volume(1, 'nova') + self.assertEqual(volume.size, 1) + self.data['volume'] = volume + # Give network time to find volume. + time.sleep(5) + + def test_002_can_attach_volume(self): + volume = self.data['volume'] + + for x in xrange(10): + if volume.status == u'available': + break + time.sleep(5) + volume.update() + else: + self.fail('cannot attach volume with state %s' % volume.status) + + volume.attach(self.data['instance'].id, self.device) + + # Volumes seems to report "available" too soon. + for x in xrange(10): + if volume.status == u'in-use': + break + time.sleep(5) + volume.update() + + self.assertEqual(volume.status, u'in-use') + + # Give instance time to recognize volume. + time.sleep(5) + + def test_003_can_mount_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + commands = [] + commands.append('mkdir -p /mnt/vol') + commands.append('mkfs.ext2 %s' % self.device) + commands.append('mount %s /mnt/vol' % self.device) + commands.append('echo success') + stdin, stdout, stderr = conn.exec_command(' && '.join(commands)) + out = stdout.read() + conn.close() + if not out.strip().endswith('success'): + self.fail('Unable to mount: %s %s' % (out, stderr.read())) + + def test_004_can_write_to_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + # FIXME(devcamcar): This doesn't fail if the volume hasn't been mounted + stdin, stdout, stderr = conn.exec_command( + 'echo hello > /mnt/vol/test.txt') + err = stderr.read() + conn.close() + if len(err) > 0: + self.fail('Unable to write to mount: %s' % (err)) + + def test_005_volume_is_correct_size(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + stdin, stdout, stderr = conn.exec_command( + "df -h | grep %s | awk {'print $2'}" % self.device) + out = stdout.read() + conn.close() + if not out.strip() == '1008M': + self.fail('Volume is not the right size: %s %s' % + (out, stderr.read())) + + def test_006_me_can_umount_volume(self): + ip = self.data['instance'].private_dns_name + conn = self.connect_ssh(ip, TEST_KEY) + stdin, stdout, stderr = conn.exec_command('umount /mnt/vol') + err = stderr.read() + conn.close() + if len(err) > 0: + self.fail('Unable to unmount: %s' % (err)) + + def test_007_me_can_detach_volume(self): + result = self.conn.detach_volume(volume_id=self.data['volume'].id) + self.assertTrue(result) + time.sleep(5) + + def test_008_me_can_delete_volume(self): + result = self.conn.delete_volume(self.data['volume'].id) + self.assertTrue(result) + + def test_999_tearDown(self): + self.conn.terminate_instances([self.data['instance'].id]) + self.conn.delete_key_pair(TEST_KEY) + + +if __name__ == "__main__": + suites = {'image': unittest.makeSuite(ImageTests), + 'instance': unittest.makeSuite(InstanceTests), + 'volume': unittest.makeSuite(VolumeTests)} + sys.exit(base.run_tests(suites)) diff --git a/tools/clean-vlans b/tools/clean-vlans index f5b0295ad..820a9dbe5 100755 --- a/tools/clean-vlans +++ b/tools/clean-vlans @@ -17,6 +17,8 @@ # License for the specific language governing permissions and limitations # under the License. +export LC_ALL=C + sudo ifconfig -a | grep br | grep -v bridge | cut -f1 -d" " | xargs -n1 -ifoo ifconfig foo down sudo ifconfig -a | grep br | grep -v bridge | cut -f1 -d" " | xargs -n1 -ifoo brctl delbr foo sudo ifconfig -a | grep vlan | grep -v vlan124 | grep -v vlan5 | cut -f1 -d" " | xargs -n1 -ifoo ifconfig foo down diff --git a/tools/pip-requires b/tools/pip-requires index 548073326..e9559521b 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -2,6 +2,7 @@ SQLAlchemy==0.6.3 pep8==0.5.0 pylint==0.19 IPy==0.70 +Cheetah==2.4.2.1 M2Crypto==0.20.2 amqplib==0.6.1 anyjson==0.2.4 @@ -20,3 +21,5 @@ mox==0.5.0 -f http://pymox.googlecode.com/files/mox-0.5.0.tar.gz greenlet==0.3.1 nose +bzr +Twisted>=10.1.0
\ No newline at end of file diff --git a/tools/setup_iptables.sh b/tools/setup_iptables.sh index 673353eb4..8be8cd812 100755 --- a/tools/setup_iptables.sh +++ b/tools/setup_iptables.sh @@ -36,7 +36,7 @@ else # NOTE(vish): This will just get the first ip in the list, so if you # have more than one eth device set up, this will fail, and # you should explicitly pass in the ip of the instance - IP=`ifconfig | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'` + IP=`LC_ALL=C ifconfig | grep -m 1 'inet addr:'| cut -d: -f2 | awk '{print $1}'` fi if [ -n "$3" ]; then |
