diff options
| author | Cerberus <matt.dietz@rackspace.com> | 2011-01-19 10:50:54 -0600 |
|---|---|---|
| committer | Cerberus <matt.dietz@rackspace.com> | 2011-01-19 10:50:54 -0600 |
| commit | 11aaf029fa2de53ca0f8a6d1a0953bb616535cbb (patch) | |
| tree | f415027e0acff284f68e7ed563ce58c4acc34ecc | |
| parent | 2c0f1d78927c14f1d155e617a066b09a00acb100 (diff) | |
| parent | 7d7fbf5dfd8a8e10f584df5d27d3479c4b2b4d3a (diff) | |
| download | nova-11aaf029fa2de53ca0f8a6d1a0953bb616535cbb.tar.gz nova-11aaf029fa2de53ca0f8a6d1a0953bb616535cbb.tar.xz nova-11aaf029fa2de53ca0f8a6d1a0953bb616535cbb.zip | |
Merge from trunk
70 files changed, 2038 insertions, 969 deletions
@@ -16,6 +16,8 @@ <jmckenty@gmail.com> <jmckenty@joshua-mckentys-macbook-pro.local> <jmckenty@gmail.com> <joshua.mckenty@nasa.gov> <justin@fathomdb.com> <justinsb@justinsb-desktop> +<masumotok@nttdata.co.jp> <root@openstack2-api> +<masumotok@nttdata.co.jp> Masumoto<masumotok@nttdata.co.jp> <mordred@inaugust.com> <mordred@hudson> <paul@openstack.org> <pvoccio@castor.local> <paul@openstack.org> <paul.voccio@rackspace.com> @@ -26,6 +26,7 @@ Josh Durgin <joshd@hq.newdream.net> Josh Kearney <josh.kearney@rackspace.com> Joshua McKenty <jmckenty@gmail.com> Justin Santa Barbara <justin@fathomdb.com> +Kei Masumoto <masumotok@nttdata.co.jp> Ken Pepple <ken.pepple@gmail.com> Koji Iida <iida.koji@lab.ntt.co.jp> Lorin Hochstein <lorin@isi.edu> @@ -34,6 +35,7 @@ Michael Gundlach <michael.gundlach@rackspace.com> Monsyne Dragon <mdragon@rackspace.com> Monty Taylor <mordred@inaugust.com> MORITA Kazutaka <morita.kazutaka@gmail.com> +Muneyuki Noguchi <noguchimn@nttdata.co.jp> Nachi Ueno <ueno.nachi@lab.ntt.co.jp> <openstack@lab.ntt.co.jp> <nati.ueno@gmail.com> <nova@u4> Paul Voccio <paul@openstack.org> Rick Clark <rick@openstack.org> diff --git a/MANIFEST.in b/MANIFEST.in index 199ce30b6..07e4dd516 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -12,6 +12,7 @@ include nova/cloudpipe/bootscript.sh include nova/cloudpipe/client.ovpn.template include nova/compute/fakevirtinstance.xml include nova/compute/interfaces.template +include nova/db/sqlalchemy/migrate_repo/migrate.cfg include nova/virt/interfaces.template include nova/virt/libvirt*.xml.template include nova/tests/CA/ diff --git a/bin/nova-api b/bin/nova-api index c7cbb3ab4..7b4fbeab1 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -34,22 +34,53 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) -from nova import api from nova import flags -from nova import utils +from nova import log as logging from nova import wsgi +logging.basicConfig() +LOG = logging.getLogger('nova.api') +LOG.setLevel(logging.DEBUG) FLAGS = flags.FLAGS -flags.DEFINE_string('osapi_host', '0.0.0.0', 'OpenStack API host') -flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port') -flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host') +API_ENDPOINTS = ['ec2', 'osapi'] -if __name__ == '__main__': - utils.default_flagfile() - FLAGS(sys.argv) + +def run_app(paste_config_file): + LOG.debug(_("Using paste.deploy config at: %s"), paste_config_file) + apps = [] + for api in API_ENDPOINTS: + config = wsgi.load_paste_configuration(paste_config_file, api) + if config is None: + LOG.debug(_("No paste configuration for app: %s"), api) + continue + LOG.debug(_("App Config: %s\n%r"), api, config) + wsgi.paste_config_to_flags(config, { + "verbose": FLAGS.verbose, + "%s_host" % api: config.get('host', '0.0.0.0'), + "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) + LOG.info(_("Running %s API"), api) + app = wsgi.load_paste_app(paste_config_file, api) + apps.append((app, getattr(FLAGS, "%s_port" % api), + getattr(FLAGS, "%s_host" % api))) + if len(apps) == 0: + LOG.error(_("No known API applications configured in %s."), + paste_config_file) + return + + # NOTE(todd): redo logging config, verbose could be set in paste config + logging.basicConfig() server = wsgi.Server() - server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host) - server.start(api.API('ec2'), FLAGS.ec2api_port, host=FLAGS.ec2api_host) + for app in apps: + server.start(*app) server.wait() + + +if __name__ == '__main__': + FLAGS(sys.argv) + conf = wsgi.paste_config_file('nova-api.conf') + if conf: + run_app(conf) + else: + LOG.error(_("No paste configuration found for: %s"), 'nova-api.conf') diff --git a/bin/nova-api-paste b/bin/nova-api-paste deleted file mode 100755 index 419f0bbdc..000000000 --- a/bin/nova-api-paste +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env python -# pylint: disable-msg=C0103 -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 United States Government as represented by the -# Administrator of the National Aeronautics and Space Administration. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Starter script for Nova API.""" - -import gettext -import os -import sys - -from paste import deploy - -# If ../nova/__init__.py exists, add ../ to Python search path, so that -# it will override what happens to be installed in /usr/(local/)lib/python... -possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), - os.pardir, - os.pardir)) -if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - sys.path.insert(0, possible_topdir) - -gettext.install('nova', unicode=1) - -from nova import flags -from nova import log as logging -from nova import wsgi - -LOG = logging.getLogger('nova.api') -LOG.setLevel(logging.DEBUG) -LOG.addHandler(logging.StreamHandler()) - -FLAGS = flags.FLAGS - -API_ENDPOINTS = ['ec2', 'openstack'] - - -def load_configuration(paste_config): - """Load the paste configuration from the config file and return it.""" - config = None - # Try each known name to get the global DEFAULTS, which will give ports - for name in API_ENDPOINTS: - try: - config = deploy.appconfig("config:%s" % paste_config, name=name) - except LookupError: - pass - if config: - verbose = config.get('verbose', None) - if verbose: - FLAGS.verbose = int(verbose) == 1 - if FLAGS.verbose: - logging.getLogger().setLevel(logging.DEBUG) - return config - LOG.debug(_("Paste config at %s has no secion for known apis"), - paste_config) - print _("Paste config at %s has no secion for any known apis") % \ - paste_config - os.exit(1) - - -def launch_api(paste_config_file, section, server, port, host): - """Launch an api server from the specified port and IP.""" - LOG.debug(_("Launching %s api on %s:%s"), section, host, port) - app = deploy.loadapp('config:%s' % paste_config_file, name=section) - server.start(app, int(port), host) - - -def run_app(paste_config_file): - LOG.debug(_("Using paste.deploy config at: %s"), configfile) - config = load_configuration(paste_config_file) - LOG.debug(_("Configuration: %r"), config) - server = wsgi.Server() - ip = config.get('host', '0.0.0.0') - for api in API_ENDPOINTS: - port = config.get("%s_port" % api, None) - if not port: - continue - host = config.get("%s_host" % api, ip) - launch_api(configfile, api, server, port, host) - LOG.debug(_("All api servers launched, now waiting")) - server.wait() - - -if __name__ == '__main__': - FLAGS(sys.argv) - configfiles = ['/etc/nova/nova-api.conf'] - if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): - configfiles.insert(0, - os.path.join(possible_topdir, 'etc', 'nova-api.conf')) - for configfile in configfiles: - if os.path.exists(configfile): - run_app(configfile) - break - else: - LOG.debug(_("Skipping missing configuration: %s"), configfile) diff --git a/bin/nova-combined b/bin/nova-combined index f932fdfd5..913c866bf 100755 --- a/bin/nova-combined +++ b/bin/nova-combined @@ -36,22 +36,20 @@ if os.path.exists(os.path.join(possible_topdir, 'nova', '__init__.py')): gettext.install('nova', unicode=1) -from nova import api from nova import flags +from nova import log as logging from nova import service from nova import utils from nova import wsgi FLAGS = flags.FLAGS -flags.DEFINE_string('osapi_host', '0.0.0.0', 'OpenStack API host') -flags.DEFINE_integer('ec2api_port', 8773, 'EC2 API port') -flags.DEFINE_string('ec2api_host', '0.0.0.0', 'EC2 API host') if __name__ == '__main__': utils.default_flagfile() FLAGS(sys.argv) + logging.basicConfig() compute = service.Service.create(binary='nova-compute') network = service.Service.create(binary='nova-network') @@ -61,7 +59,22 @@ if __name__ == '__main__': service.serve(compute, network, volume, scheduler) - server = wsgi.Server() - server.start(api.API('os'), FLAGS.osapi_port, host=FLAGS.osapi_host) - server.start(api.API('ec2'), FLAGS.ec2api_port, host=FLAGS.ec2api_host) - server.wait() + apps = [] + paste_config_file = wsgi.paste_config_file('nova-api.conf') + for api in ['osapi', 'ec2']: + config = wsgi.load_paste_configuration(paste_config_file, api) + if config is None: + continue + wsgi.paste_config_to_flags(config, { + "verbose": FLAGS.verbose, + "%s_host" % api: config.get('host', '0.0.0.0'), + "%s_port" % api: getattr(FLAGS, "%s_port" % api)}) + app = wsgi.load_paste_app(paste_config_file, api) + apps.append((app, getattr(FLAGS, "%s_port" % api), + getattr(FLAGS, "%s_host" % api))) + if len(apps) > 0: + logging.basicConfig() + server = wsgi.Server() + for app in apps: + server.start(*app) + server.wait() diff --git a/bin/nova-manage b/bin/nova-manage index b5842b595..d0901ddfc 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -82,6 +82,7 @@ from nova import quota from nova import utils from nova.auth import manager from nova.cloudpipe import pipelib +from nova.db import migration logging.basicConfig() @@ -519,6 +520,21 @@ class LogCommands(object): print re.sub('#012', "\n", "\n".join(lines)) +class DbCommands(object): + """Class for managing the database.""" + + def __init__(self): + pass + + def sync(self, version=None): + """Sync the database up to the most recent version.""" + return migration.db_sync(version) + + def version(self): + """Print the current database version.""" + print migration.db_version() + + CATEGORIES = [ ('user', UserCommands), ('project', ProjectCommands), @@ -528,7 +544,8 @@ CATEGORIES = [ ('floating', FloatingIpCommands), ('network', NetworkCommands), ('service', ServiceCommands), - ('log', LogCommands)] + ('log', LogCommands), + ('db', DbCommands)] def lazy_match(name, key_value_tuples): diff --git a/etc/nova-api.conf b/etc/nova-api.conf index c5dd0aaec..4873e465d 100644 --- a/etc/nova-api.conf +++ b/etc/nova-api.conf @@ -1,9 +1,5 @@ [DEFAULT] verbose = 1 -ec2_port = 8773 -ec2_address = 0.0.0.0 -openstack_port = 8774 -openstack_address = 0.0.0.0 ####### # EC2 # @@ -12,52 +8,80 @@ openstack_address = 0.0.0.0 [composite:ec2] use = egg:Paste#urlmap /: ec2versions -/services: ec2api +/services/Cloud: ec2cloud +/services/Admin: ec2admin /latest: ec2metadata -/200: ec2metadata +/20: ec2metadata /1.0: ec2metadata -[pipeline:ec2api] -pipeline = authenticate router authorizer ec2executor +[pipeline:ec2cloud] +pipeline = logrequest authenticate cloudrequest authorizer ec2executor +#pipeline = logrequest ec2lockout authenticate cloudrequest authorizer ec2executor + +[pipeline:ec2admin] +pipeline = logrequest authenticate adminrequest authorizer ec2executor + +[pipeline:ec2metadata] +pipeline = logrequest ec2md + +[pipeline:ec2versions] +pipeline = logrequest ec2ver + +[filter:logrequest] +paste.filter_factory = nova.api.ec2:RequestLogging.factory + +[filter:ec2lockout] +paste.filter_factory = nova.api.ec2:Lockout.factory [filter:authenticate] -paste.filter_factory = nova.api.ec2:authenticate_factory +paste.filter_factory = nova.api.ec2:Authenticate.factory + +[filter:cloudrequest] +controller = nova.api.ec2.cloud.CloudController +paste.filter_factory = nova.api.ec2:Requestify.factory -[filter:router] -paste.filter_factory = nova.api.ec2:router_factory +[filter:adminrequest] +controller = nova.api.ec2.admin.AdminController +paste.filter_factory = nova.api.ec2:Requestify.factory [filter:authorizer] -paste.filter_factory = nova.api.ec2:authorizer_factory +paste.filter_factory = nova.api.ec2:Authorizer.factory [app:ec2executor] -paste.app_factory = nova.api.ec2:executor_factory +paste.app_factory = nova.api.ec2:Executor.factory -[app:ec2versions] -paste.app_factory = nova.api.ec2:versions_factory +[app:ec2ver] +paste.app_factory = nova.api.ec2:Versions.factory -[app:ec2metadata] -paste.app_factory = nova.api.ec2.metadatarequesthandler:metadata_factory +[app:ec2md] +paste.app_factory = nova.api.ec2.metadatarequesthandler:MetadataRequestHandler.factory ############# # Openstack # ############# -[composite:openstack] +[composite:osapi] use = egg:Paste#urlmap /: osversions /v1.0: openstackapi [pipeline:openstackapi] -pipeline = auth ratelimit osapi +pipeline = faultwrap auth ratelimit osapiapp + +[filter:faultwrap] +paste.filter_factory = nova.api.openstack:FaultWrapper.factory [filter:auth] -paste.filter_factory = nova.api.openstack.auth:auth_factory +paste.filter_factory = nova.api.openstack.auth:AuthMiddleware.factory [filter:ratelimit] -paste.filter_factory = nova.api.openstack.ratelimiting:ratelimit_factory +paste.filter_factory = nova.api.openstack.ratelimiting:RateLimitingMiddleware.factory + +[app:osapiapp] +paste.app_factory = nova.api.openstack:APIRouter.factory -[app:osapi] -paste.app_factory = nova.api.openstack:router_factory +[pipeline:osversions] +pipeline = faultwrap osversionapp -[app:osversions] -paste.app_factory = nova.api.openstack:versions_factory +[app:osversionapp] +paste.app_factory = nova.api.openstack:Versions.factory diff --git a/nova/api/__init__.py b/nova/api/__init__.py index 803470570..0fedbbfad 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -15,96 +15,5 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -""" -Root WSGI middleware for all API controllers. -**Related Flags** - -:osapi_subdomain: subdomain running the OpenStack API (default: api) -:ec2api_subdomain: subdomain running the EC2 API (default: ec2) - -""" - -import routes -import webob.dec - -from nova import flags -from nova import wsgi -from nova.api import ec2 -from nova.api import openstack -from nova.api.ec2 import metadatarequesthandler - - -flags.DEFINE_string('osapi_subdomain', 'api', - 'subdomain running the OpenStack API') -flags.DEFINE_string('ec2api_subdomain', 'ec2', - 'subdomain running the EC2 API') - -FLAGS = flags.FLAGS - - -class API(wsgi.Router): - """Routes top-level requests to the appropriate controller.""" - - def __init__(self, default_api): - osapi_subdomain = {'sub_domain': [FLAGS.osapi_subdomain]} - ec2api_subdomain = {'sub_domain': [FLAGS.ec2api_subdomain]} - if default_api == 'os': - osapi_subdomain = {} - elif default_api == 'ec2': - ec2api_subdomain = {} - mapper = routes.Mapper() - mapper.sub_domains = True - - mapper.connect("/", controller=self.osapi_versions, - conditions=osapi_subdomain) - mapper.connect("/v1.0/{path_info:.*}", controller=openstack.API(), - conditions=osapi_subdomain) - - mapper.connect("/", controller=self.ec2api_versions, - conditions=ec2api_subdomain) - mapper.connect("/services/{path_info:.*}", controller=ec2.API(), - conditions=ec2api_subdomain) - mrh = metadatarequesthandler.MetadataRequestHandler() - for s in ['/latest', - '/2009-04-04', - '/2008-09-01', - '/2008-02-01', - '/2007-12-15', - '/2007-10-10', - '/2007-08-29', - '/2007-03-01', - '/2007-01-19', - '/1.0']: - mapper.connect('%s/{path_info:.*}' % s, controller=mrh, - conditions=ec2api_subdomain) - - super(API, self).__init__(mapper) - - @webob.dec.wsgify - def osapi_versions(self, req): - """Respond to a request for all OpenStack API versions.""" - response = { - "versions": [ - dict(status="CURRENT", id="v1.0")]} - metadata = { - "application/xml": { - "attributes": dict(version=["status", "id"])}} - return wsgi.Serializer(req.environ, metadata).to_content_type(response) - - @webob.dec.wsgify - def ec2api_versions(self, req): - """Respond to a request for all EC2 versions.""" - # available api versions - versions = [ - '1.0', - '2007-01-19', - '2007-03-01', - '2007-08-29', - '2007-10-10', - '2007-12-15', - '2008-02-01', - '2008-09-01', - '2009-04-04', - ] - return ''.join('%s\n' % v for v in versions) +"""No-op __init__ for directory full of api goodies.""" diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 2fa1f636c..238cb0f38 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -30,10 +30,9 @@ from nova import context from nova import exception from nova import flags from nova import log as logging +from nova import utils from nova import wsgi from nova.api.ec2 import apirequest -from nova.api.ec2 import admin -from nova.api.ec2 import cloud from nova.auth import manager @@ -42,8 +41,6 @@ LOG = logging.getLogger("nova.api") flags.DEFINE_boolean('use_forwarded_for', False, 'Treat X-Forwarded-For as the canonical remote address. ' 'Only enable this if you have a sanitizing proxy.') -flags.DEFINE_boolean('use_lockout', False, - 'Whether or not to use lockout middleware.') flags.DEFINE_integer('lockout_attempts', 5, 'Number of failed auths before lockout.') flags.DEFINE_integer('lockout_minutes', 15, @@ -54,13 +51,8 @@ flags.DEFINE_list('lockout_memcached_servers', None, 'Memcached servers or None for in process cache.') -class API(wsgi.Middleware): - """Routing for all EC2 API requests.""" - - def __init__(self): - self.application = Authenticate(Router(Authorizer(Executor()))) - if FLAGS.use_lockout: - self.application = Lockout(self.application) +class RequestLogging(wsgi.Middleware): + """Access-Log akin logging for all EC2 API requests.""" @webob.dec.wsgify def __call__(self, req): @@ -192,26 +184,14 @@ class Authenticate(wsgi.Middleware): return self.application -class Router(wsgi.Middleware): - - """Add ec2.'controller', .'action', and .'action_args' to WSGI environ.""" +class Requestify(wsgi.Middleware): - def __init__(self, application): - super(Router, self).__init__(application) - self.map = routes.Mapper() - self.map.connect("/{controller_name}/") - self.controllers = dict(Cloud=cloud.CloudController(), - Admin=admin.AdminController()) + def __init__(self, app, controller): + super(Requestify, self).__init__(app) + self.controller = utils.import_class(controller)() @webob.dec.wsgify def __call__(self, req): - # Obtain the appropriate controller and action for this request. - try: - match = self.map.match(req.path_info) - controller_name = match['controller_name'] - controller = self.controllers[controller_name] - except: - raise webob.exc.HTTPNotFound() non_args = ['Action', 'Signature', 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Version', 'Timestamp'] args = dict(req.params) @@ -229,8 +209,8 @@ class Router(wsgi.Middleware): LOG.debug(_('arg: %s\t\tval: %s'), key, value) # Success! - req.environ['ec2.controller'] = controller - req.environ['ec2.action'] = action + api_request = apirequest.APIRequest(self.controller, action, args) + req.environ['ec2.request'] = api_request req.environ['ec2.action_args'] = args return self.application @@ -291,16 +271,14 @@ class Authorizer(wsgi.Middleware): @webob.dec.wsgify def __call__(self, req): context = req.environ['ec2.context'] - controller_name = req.environ['ec2.controller'].__class__.__name__ - action = req.environ['ec2.action'] - allowed_roles = self.action_roles[controller_name].get(action, - ['none']) + controller = req.environ['ec2.request'].controller.__class__.__name__ + action = req.environ['ec2.request'].action + allowed_roles = self.action_roles[controller].get(action, ['none']) if self._matches_any_role(context, allowed_roles): return self.application else: LOG.audit(_("Unauthorized request for controller=%s " - "and action=%s"), controller_name, action, - context=context) + "and action=%s"), controller, action, context=context) raise webob.exc.HTTPUnauthorized() def _matches_any_role(self, context, roles): @@ -327,14 +305,10 @@ class Executor(wsgi.Application): @webob.dec.wsgify def __call__(self, req): context = req.environ['ec2.context'] - controller = req.environ['ec2.controller'] - action = req.environ['ec2.action'] - args = req.environ['ec2.action_args'] - - api_request = apirequest.APIRequest(controller, action) + api_request = req.environ['ec2.request'] result = None try: - result = api_request.send(context, **args) + result = api_request.invoke(context) except exception.NotFound as ex: LOG.info(_('NotFound raised: %s'), str(ex), context=context) return self._error(req, context, type(ex).__name__, str(ex)) @@ -391,29 +365,3 @@ class Versions(wsgi.Application): '2009-04-04', ] return ''.join('%s\n' % v for v in versions) - - -def authenticate_factory(global_args, **local_args): - def authenticator(app): - return Authenticate(app) - return authenticator - - -def router_factory(global_args, **local_args): - def router(app): - return Router(app) - return router - - -def authorizer_factory(global_args, **local_args): - def authorizer(app): - return Authorizer(app) - return authorizer - - -def executor_factory(global_args, **local_args): - return Executor() - - -def versions_factory(global_args, **local_args): - return Versions() diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py index d0b417db1..78576470a 100644 --- a/nova/api/ec2/apirequest.py +++ b/nova/api/ec2/apirequest.py @@ -83,11 +83,12 @@ def _try_convert(value): class APIRequest(object): - def __init__(self, controller, action): + def __init__(self, controller, action, args): self.controller = controller self.action = action + self.args = args - def send(self, context, **kwargs): + def invoke(self, context): try: method = getattr(self.controller, _camelcase_to_underscore(self.action)) @@ -100,7 +101,7 @@ class APIRequest(object): raise Exception(_error) args = {} - for key, value in kwargs.items(): + for key, value in self.args.items(): parts = key.split(".") key = _camelcase_to_underscore(parts[0]) if isinstance(value, str) or isinstance(value, unicode): diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index a1ae70fb0..c94540793 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -254,15 +254,15 @@ class CloudController(object): name, _sep, host = region.partition('=') endpoint = '%s://%s:%s%s' % (FLAGS.ec2_prefix, host, - FLAGS.cc_port, + FLAGS.ec2_port, FLAGS.ec2_suffix) regions.append({'regionName': name, 'regionEndpoint': endpoint}) else: regions = [{'regionName': 'nova', 'regionEndpoint': '%s://%s:%s%s' % (FLAGS.ec2_prefix, - FLAGS.cc_host, - FLAGS.cc_port, + FLAGS.ec2_host, + FLAGS.ec2_port, FLAGS.ec2_suffix)}] return {'regionInfo': regions} @@ -729,7 +729,7 @@ class CloudController(object): ec2_id = None if (floating_ip_ref['fixed_ip'] and floating_ip_ref['fixed_ip']['instance']): - instance_id = floating_ip_ref['fixed_ip']['instance']['ec2_id'] + instance_id = floating_ip_ref['fixed_ip']['instance']['id'] ec2_id = id_to_ec2_id(instance_id) address_rv = {'public_ip': address, 'instance_id': ec2_id} diff --git a/nova/api/ec2/metadatarequesthandler.py b/nova/api/ec2/metadatarequesthandler.py index 848f0b034..6fb441656 100644 --- a/nova/api/ec2/metadatarequesthandler.py +++ b/nova/api/ec2/metadatarequesthandler.py @@ -23,6 +23,7 @@ import webob.exc from nova import log as logging from nova import flags +from nova import wsgi from nova.api.ec2 import cloud @@ -30,7 +31,7 @@ LOG = logging.getLogger('nova.api.ec2.metadata') FLAGS = flags.FLAGS -class MetadataRequestHandler(object): +class MetadataRequestHandler(wsgi.Application): """Serve metadata from the EC2 API.""" def print_data(self, data): @@ -78,7 +79,3 @@ class MetadataRequestHandler(object): if data is None: raise webob.exc.HTTPNotFound() return self.print_data(data) - - -def metadata_factory(global_args, **local_args): - return MetadataRequestHandler() diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index f96e2af91..f2caac483 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -23,11 +23,9 @@ WSGI middleware for OpenStack API controllers. import routes import webob.dec import webob.exc -import webob from nova import flags from nova import log as logging -from nova import utils from nova import wsgi from nova.api.openstack import faults from nova.api.openstack import backup_schedules @@ -40,32 +38,16 @@ from nova.api.openstack import shared_ip_groups LOG = logging.getLogger('nova.api.openstack') FLAGS = flags.FLAGS -flags.DEFINE_string('os_api_auth', - 'nova.api.openstack.auth.AuthMiddleware', - 'The auth mechanism to use for the OpenStack API implemenation') - -flags.DEFINE_string('os_api_ratelimiting', - 'nova.api.openstack.ratelimiting.RateLimitingMiddleware', - 'Default ratelimiting implementation for the Openstack API') - flags.DEFINE_string('os_krm_mapping_file', 'krm_mapping.json', 'Location of OpenStack Flavor/OS:EC2 Kernel/Ramdisk/Machine JSON file.') - flags.DEFINE_bool('allow_admin_api', False, 'When True, this API service will accept admin operations.') -class API(wsgi.Middleware): - """WSGI entry point for all OpenStack API requests.""" - - def __init__(self): - auth_middleware = utils.import_class(FLAGS.os_api_auth) - ratelimiting_middleware = \ - utils.import_class(FLAGS.os_api_ratelimiting) - app = auth_middleware(ratelimiting_middleware(APIRouter())) - super(API, self).__init__(app) +class FaultWrapper(wsgi.Middleware): + """Calls down the middleware stack, making exceptions into faults.""" @webob.dec.wsgify def __call__(self, req): @@ -83,6 +65,11 @@ class APIRouter(wsgi.Router): and method. """ + @classmethod + def factory(cls, global_config, **local_config): + """Simple paste factory, :class:`nova.wsgi.Router` doesn't have one""" + return cls() + def __init__(self): mapper = routes.Mapper() @@ -132,11 +119,3 @@ class Versions(wsgi.Application): "application/xml": { "attributes": dict(version=["status", "id"])}} return wsgi.Serializer(req.environ, metadata).to_content_type(response) - - -def router_factory(global_cof, **local_conf): - return APIRouter() - - -def versions_factory(global_conf, **local_conf): - return Versions() diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 00e817c8d..1dfdd5318 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -134,9 +134,3 @@ class AuthMiddleware(wsgi.Middleware): token = self.db.auth_create_token(ctxt, token_dict) return token, user return None, None - - -def auth_factory(global_conf, **local_conf): - def auth(app): - return AuthMiddleware(app) - return auth diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index a5f55a489..9d56bc508 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -78,7 +78,14 @@ def _translate_status(item): 'decrypting': 'preparing', 'untarring': 'saving', 'available': 'active'} - item['status'] = status_mapping[item['status']] + try: + item['status'] = status_mapping[item['status']] + except KeyError: + # TODO(sirp): Performing translation of status (if necessary) here for + # now. Perhaps this should really be done in EC2 API and + # S3ImageService + pass + return item @@ -92,9 +99,11 @@ def _filter_keys(item, keys): def _convert_image_id_to_hash(image): - image_id = abs(hash(image['imageId'])) - image['imageId'] = image_id - image['id'] = image_id + if 'imageId' in image: + # Convert EC2-style ID (i-blah) to Rackspace-style (int) + image_id = abs(hash(image['imageId'])) + image['imageId'] = image_id + image['id'] = image_id class Controller(wsgi.Controller): @@ -147,7 +156,11 @@ class Controller(wsgi.Controller): env = self._deserialize(req.body, req) instance_id = env["image"]["serverId"] name = env["image"]["name"] - return compute.API().snapshot(context, instance_id, name) + + image_meta = compute.API().snapshot( + context, instance_id, name) + + return dict(image=image_meta) def update(self, req, id): # Users may not modify public images, and that's all that diff --git a/nova/api/openstack/ratelimiting/__init__.py b/nova/api/openstack/ratelimiting/__init__.py index 81b83142f..cbb4b897e 100644 --- a/nova/api/openstack/ratelimiting/__init__.py +++ b/nova/api/openstack/ratelimiting/__init__.py @@ -219,9 +219,3 @@ class WSGIAppProxy(object): # No delay return None return float(resp.getheader('X-Wait-Seconds')) - - -def ratelimit_factory(global_conf, **local_conf): - def rl(app): - return RateLimitingMiddleware(app) - return rl diff --git a/nova/auth/manager.py b/nova/auth/manager.py index de18a3838..1652e24e1 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -682,7 +682,7 @@ class AuthManager(object): region, _sep, region_host = item.partition("=") regions[region] = region_host else: - regions = {'nova': FLAGS.cc_host} + regions = {'nova': FLAGS.ec2_host} for region, host in regions.iteritems(): rc = self.__generate_rc(user, pid, @@ -727,28 +727,28 @@ class AuthManager(object): def __generate_rc(user, pid, use_dmz=True, host=None): """Generate rc file for user""" if use_dmz: - cc_host = FLAGS.cc_dmz + ec2_host = FLAGS.ec2_dmz_host else: - cc_host = FLAGS.cc_host + ec2_host = FLAGS.ec2_host # NOTE(vish): Always use the dmz since it is used from inside the # instance s3_host = FLAGS.s3_dmz if host: s3_host = host - cc_host = host + ec2_host = host rc = open(FLAGS.credentials_template).read() rc = rc % {'access': user.access, 'project': pid, 'secret': user.secret, - 'ec2': '%s://%s:%s%s' % (FLAGS.ec2_prefix, - cc_host, - FLAGS.cc_port, - FLAGS.ec2_suffix), + 'ec2': '%s://%s:%s%s' % (FLAGS.ec2_scheme, + ec2_host, + FLAGS.ec2_port, + FLAGS.ec2_path), 's3': 'http://%s:%s' % (s3_host, FLAGS.s3_port), - 'os': '%s://%s:%s%s' % (FLAGS.os_prefix, - cc_host, + 'os': '%s://%s:%s%s' % (FLAGS.osapi_scheme, + ec2_host, FLAGS.osapi_port, - FLAGS.os_suffix), + FLAGS.osapi_path), 'user': user.name, 'nova': FLAGS.ca_file, 'cert': FLAGS.credential_cert_file, diff --git a/nova/cloudpipe/pipelib.py b/nova/cloudpipe/pipelib.py index 8aefd341f..dc6f55af2 100644 --- a/nova/cloudpipe/pipelib.py +++ b/nova/cloudpipe/pipelib.py @@ -68,8 +68,8 @@ class CloudPipe(object): shellfile = open(FLAGS.boot_script_template, "r") s = string.Template(shellfile.read()) shellfile.close() - boot_script = s.substitute(cc_dmz=FLAGS.cc_dmz, - cc_port=FLAGS.cc_port, + boot_script = s.substitute(cc_dmz=FLAGS.ec2_dmz_host, + cc_port=FLAGS.ec2_port, dmz_net=FLAGS.dmz_net, dmz_mask=FLAGS.dmz_mask, num_vpn=FLAGS.cnt_vpn_clients) diff --git a/nova/compute/api.py b/nova/compute/api.py index 22d5964c6..ae760fd47 100644 --- a/nova/compute/api.py +++ b/nova/compute/api.py @@ -335,27 +335,55 @@ class API(base.Base): project_id) return self.db.instance_get_all(context) - def _cast_compute_message(self, method, context, instance_id, host=None): - """Generic handler for RPC casts to compute.""" + def _cast_compute_message(self, method, context, instance_id, host=None, + params=None): + """Generic handler for RPC casts to compute. + + :param params: Optional dictionary of arguments to be passed to the + compute worker + + :retval None + """ + if not params: + params = {} if not host: instance = self.get(context, instance_id) host = instance['host'] queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) - kwargs = {'method': method, 'args': {'instance_id': instance_id}} + params['instance_id'] = instance_id + kwargs = {'method': method, 'args': params} rpc.cast(context, queue, kwargs) - def _call_compute_message(self, method, context, instance_id, host=None): - """Generic handler for RPC calls to compute.""" + def _call_compute_message(self, method, context, instance_id, host=None, + params=None): + """Generic handler for RPC calls to compute. + + :param params: Optional dictionary of arguments to be passed to the + compute worker + + :retval: Result returned by compute worker + """ + if not params: + params = {} if not host: instance = self.get(context, instance_id) host = instance["host"] queue = self.db.queue_get_for(context, FLAGS.compute_topic, host) - kwargs = {"method": method, "args": {"instance_id": instance_id}} + params['instance_id'] = instance_id + kwargs = {'method': method, 'args': params} return rpc.call(context, queue, kwargs) def snapshot(self, context, instance_id, name): - """Snapshot the given instance.""" - self._cast_compute_message('snapshot_instance', context, instance_id) + """Snapshot the given instance. + + :retval: A dict containing image metadata + """ + data = {'name': name, 'is_public': False} + image_meta = self.image_service.create(context, data) + params = {'image_id': image_meta['id']} + self._cast_compute_message('snapshot_instance', context, instance_id, + params=params) + return image_meta def reboot(self, context, instance_id): """Reboot the given instance.""" diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 714cec209..7f0641d44 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -294,7 +294,7 @@ class ComputeManager(manager.Manager): self._update_state(context, instance_id) @exception.wrap_exception - def snapshot_instance(self, context, instance_id, name): + def snapshot_instance(self, context, instance_id, image_id): """Snapshot an instance on this server.""" context = context.elevated() instance_ref = self.db.instance_get(context, instance_id) @@ -311,7 +311,7 @@ class ComputeManager(manager.Manager): 'instance: %s (state: %s excepted: %s)'), instance_id, instance_ref['state'], power_state.RUNNING) - self.driver.snapshot(instance_ref, name) + self.driver.snapshot(instance_ref, image_id) @exception.wrap_exception @checks_instance_lock diff --git a/tools/iscsidev.sh b/nova/db/migration.py index 6f5b572df..e54b90cd8 100755..100644 --- a/tools/iscsidev.sh +++ b/nova/db/migration.py @@ -1,4 +1,4 @@ -#!/bin/sh +# vim: tabstop=4 shiftwidth=4 softtabstop=4 # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. @@ -15,27 +15,24 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +"""Database setup and migration commands.""" -# NOTE(vish): This script helps udev create common names for discovered iscsi -# volumes under /dev/iscsi. To use it, create /dev/iscsi and add -# a file to /etc/udev/rules.d like so: -# mkdir /dev/iscsi -# echo 'KERNEL=="sd*", BUS=="scsi", PROGRAM="/path/to/iscsidev.sh -# %b",SYMLINK+="iscsi/%c%n"' > /etc/udev/rules.d/55-openiscsi.rules +from nova import flags +from nova import utils -BUS=${1} -HOST=${BUS%%:*} +FLAGS = flags.FLAGS +flags.DECLARE('db_backend', 'nova.db.api') -if [ ! -e /sys/class/iscsi_host ]; then - exit 1 -fi -file="/sys/class/iscsi_host/host${HOST}/device/session*/iscsi_session*/session*/targetname" +IMPL = utils.LazyPluggable(FLAGS['db_backend'], + sqlalchemy='nova.db.sqlalchemy.migration') -target_name=$(cat ${file}) -if [ -z "${target_name}" ]; then - exit 1 -fi +def db_sync(version=None): + """Migrate the database to `version` or the most recent version.""" + return IMPL.db_sync(version=version) -echo "${target_name##*:}" + +def db_version(): + """Display the current database version.""" + return IMPL.db_version() diff --git a/nova/db/sqlalchemy/__init__.py b/nova/db/sqlalchemy/__init__.py index 501373942..747015af5 100644 --- a/nova/db/sqlalchemy/__init__.py +++ b/nova/db/sqlalchemy/__init__.py @@ -15,31 +15,3 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -""" -SQLAlchemy database backend -""" -import time - -from sqlalchemy.exc import OperationalError - -from nova import flags -from nova import log as logging -from nova.db.sqlalchemy import models - - -FLAGS = flags.FLAGS -LOG = logging.getLogger('nova.db.sqlalchemy') - - -for i in xrange(FLAGS.sql_max_retries): - if i > 0: - time.sleep(FLAGS.sql_retry_interval) - - try: - models.register_models() - break - except OperationalError: - LOG.exception(_("Data store %s is unreachable." - " Trying again in %d seconds."), - FLAGS.sql_connection, FLAGS.sql_retry_interval) diff --git a/nova/db/sqlalchemy/migrate_repo/README b/nova/db/sqlalchemy/migrate_repo/README new file mode 100644 index 000000000..6218f8cac --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/README @@ -0,0 +1,4 @@ +This is a database migration repository. + +More information at +http://code.google.com/p/sqlalchemy-migrate/ diff --git a/nova/db/sqlalchemy/migrate_repo/__init__.py b/nova/db/sqlalchemy/migrate_repo/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/__init__.py diff --git a/nova/db/sqlalchemy/migrate_repo/manage.py b/nova/db/sqlalchemy/migrate_repo/manage.py new file mode 100644 index 000000000..09e340f44 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/manage.py @@ -0,0 +1,4 @@ +#!/usr/bin/env python +from migrate.versioning.shell import main +if __name__ == '__main__': + main(debug='False', repository='.') diff --git a/nova/db/sqlalchemy/migrate_repo/migrate.cfg b/nova/db/sqlalchemy/migrate_repo/migrate.cfg new file mode 100644 index 000000000..2c75fb763 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/migrate.cfg @@ -0,0 +1,20 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=nova + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] diff --git a/nova/db/sqlalchemy/migrate_repo/versions/001_austin.py b/nova/db/sqlalchemy/migrate_repo/versions/001_austin.py new file mode 100644 index 000000000..a312a7190 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/001_austin.py @@ -0,0 +1,547 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +## Table code mostly autogenerated by genmodel.py +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + + +auth_tokens = Table('auth_tokens', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('token_hash', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + primary_key=True, + nullable=False), + Column('user_id', Integer()), + Column('server_manageent_url', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('storage_url', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('cdn_management_url', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + + +export_devices = Table('export_devices', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('shelf_id', Integer()), + Column('blade_id', Integer()), + Column('volume_id', + Integer(), + ForeignKey('volumes.id'), + nullable=True), + ) + + +fixed_ips = Table('fixed_ips', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('address', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('network_id', + Integer(), + ForeignKey('networks.id'), + nullable=True), + Column('instance_id', + Integer(), + ForeignKey('instances.id'), + nullable=True), + Column('allocated', Boolean(create_constraint=True, name=None)), + Column('leased', Boolean(create_constraint=True, name=None)), + Column('reserved', Boolean(create_constraint=True, name=None)), + ) + + +floating_ips = Table('floating_ips', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('address', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('fixed_ip_id', + Integer(), + ForeignKey('fixed_ips.id'), + nullable=True), + Column('project_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('host', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + + +instances = Table('instances', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('internal_id', Integer()), + Column('admin_pass', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('user_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('project_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('image_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('kernel_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('ramdisk_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('launch_index', Integer()), + Column('key_name', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('key_data', + Text(length=None, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('state', Integer()), + Column('state_description', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('memory_mb', Integer()), + Column('vcpus', Integer()), + Column('local_gb', Integer()), + Column('hostname', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('host', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('instance_type', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('user_data', + Text(length=None, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('reservation_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('mac_address', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('scheduled_at', DateTime(timezone=False)), + Column('launched_at', DateTime(timezone=False)), + Column('terminated_at', DateTime(timezone=False)), + Column('display_name', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('display_description', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + + +iscsi_targets = Table('iscsi_targets', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('target_num', Integer()), + Column('host', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('volume_id', + Integer(), + ForeignKey('volumes.id'), + nullable=True), + ) + + +key_pairs = Table('key_pairs', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('name', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('user_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('fingerprint', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('public_key', + Text(length=None, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + + +networks = Table('networks', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('injected', Boolean(create_constraint=True, name=None)), + Column('cidr', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('netmask', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('bridge', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('gateway', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('broadcast', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('dns', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('vlan', Integer()), + Column('vpn_public_address', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('vpn_public_port', Integer()), + Column('vpn_private_address', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('dhcp_start', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('project_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('host', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + + +projects = Table('projects', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + primary_key=True, + nullable=False), + Column('name', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('description', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('project_manager', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + ForeignKey('users.id')), + ) + + +quotas = Table('quotas', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('project_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('instances', Integer()), + Column('cores', Integer()), + Column('volumes', Integer()), + Column('gigabytes', Integer()), + Column('floating_ips', Integer()), + ) + + +security_groups = Table('security_groups', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('name', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('description', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('user_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('project_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + + +security_group_inst_assoc = Table('security_group_instance_association', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('security_group_id', + Integer(), + ForeignKey('security_groups.id')), + Column('instance_id', Integer(), ForeignKey('instances.id')), + ) + + +security_group_rules = Table('security_group_rules', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('parent_group_id', + Integer(), + ForeignKey('security_groups.id')), + Column('protocol', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('from_port', Integer()), + Column('to_port', Integer()), + Column('cidr', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('group_id', + Integer(), + ForeignKey('security_groups.id')), + ) + + +services = Table('services', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('host', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('binary', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('topic', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('report_count', Integer(), nullable=False), + Column('disabled', Boolean(create_constraint=True, name=None)), + ) + + +users = Table('users', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + primary_key=True, + nullable=False), + Column('name', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('access_key', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('secret_key', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('is_admin', Boolean(create_constraint=True, name=None)), + ) + + +user_project_association = Table('user_project_association', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('user_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + ForeignKey('users.id'), + primary_key=True, + nullable=False), + Column('project_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + ForeignKey('projects.id'), + primary_key=True, + nullable=False), + ) + + +user_project_role_association = Table('user_project_role_association', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('user_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + primary_key=True, + nullable=False), + Column('project_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + primary_key=True, + nullable=False), + Column('role', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + primary_key=True, + nullable=False), + ForeignKeyConstraint(['user_id', + 'project_id'], + ['user_project_association.user_id', + 'user_project_association.project_id']), + ) + + +user_role_association = Table('user_role_association', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('user_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + ForeignKey('users.id'), + primary_key=True, + nullable=False), + Column('role', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + primary_key=True, + nullable=False), + ) + + +volumes = Table('volumes', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('ec2_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('user_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('project_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('host', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('size', Integer()), + Column('availability_zone', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('instance_id', + Integer(), + ForeignKey('instances.id'), + nullable=True), + Column('mountpoint', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('attach_time', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('status', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('attach_status', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('scheduled_at', DateTime(timezone=False)), + Column('launched_at', DateTime(timezone=False)), + Column('terminated_at', DateTime(timezone=False)), + Column('display_name', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('display_description', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + + for table in (auth_tokens, export_devices, fixed_ips, floating_ips, + instances, iscsi_targets, key_pairs, networks, + projects, quotas, security_groups, security_group_inst_assoc, + security_group_rules, services, users, + user_project_association, user_project_role_association, + user_role_association, volumes): + try: + table.create() + except Exception: + logging.info(repr(table)) + logging.exception('Exception while creating table') + raise + + +def downgrade(migrate_engine): + # Operations to reverse the above upgrade go here. + for table in (auth_tokens, export_devices, fixed_ips, floating_ips, + instances, iscsi_targets, key_pairs, networks, + projects, quotas, security_groups, security_group_inst_assoc, + security_group_rules, services, users, + user_project_association, user_project_role_association, + user_role_association, volumes): + table.drop() diff --git a/nova/db/sqlalchemy/migrate_repo/versions/002_bexar.py b/nova/db/sqlalchemy/migrate_repo/versions/002_bexar.py new file mode 100644 index 000000000..bd3a3e6f8 --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/002_bexar.py @@ -0,0 +1,209 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from sqlalchemy import * +from migrate import * + +from nova import log as logging + + +meta = MetaData() + + +# Just for the ForeignKey and column creation to succeed, these are not the +# actual definitions of instances or services. +instances = Table('instances', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +services = Table('services', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +networks = Table('networks', meta, + Column('id', Integer(), primary_key=True, nullable=False), + ) + + +# +# New Tables +# +certificates = Table('certificates', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('user_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('project_id', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('file_name', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + + +consoles = Table('consoles', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('instance_name', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('instance_id', Integer()), + Column('password', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('port', Integer(), nullable=True), + Column('pool_id', + Integer(), + ForeignKey('console_pools.id')), + ) + + +console_pools = Table('console_pools', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('address', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('username', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('password', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('console_type', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('public_hostname', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('host', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('compute_host', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + + +instance_actions = Table('instance_actions', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('id', Integer(), primary_key=True, nullable=False), + Column('instance_id', + Integer(), + ForeignKey('instances.id')), + Column('action', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('error', + Text(length=None, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + + +# +# Tables to alter +# +auth_tokens = Table('auth_tokens', meta, + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('deleted_at', DateTime(timezone=False)), + Column('deleted', Boolean(create_constraint=True, name=None)), + Column('token_hash', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False), + primary_key=True, + nullable=False), + Column('user_id', Integer()), + Column('server_manageent_url', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('storage_url', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + Column('cdn_management_url', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + + +instances_availability_zone = Column( + 'availability_zone', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)) + + +instances_locked = Column('locked', + Boolean(create_constraint=True, name=None)) + + +networks_cidr_v6 = Column( + 'cidr_v6', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)) + +networks_ra_server = Column( + 'ra_server', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)) + + +services_availability_zone = Column( + 'availability_zone', + String(length=255, convert_unicode=False, assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)) + + +def upgrade(migrate_engine): + # Upgrade operations go here. Don't create your own engine; + # bind migrate_engine to your metadata + meta.bind = migrate_engine + for table in (certificates, consoles, console_pools, instance_actions): + try: + table.create() + except Exception: + logging.info(repr(table)) + logging.exception('Exception while creating table') + raise + + auth_tokens.c.user_id.alter(type=String(length=255, + convert_unicode=False, + assert_unicode=None, + unicode_error=None, + _warn_on_bytestring=False)) + + instances.create_column(instances_availability_zone) + instances.create_column(instances_locked) + networks.create_column(networks_cidr_v6) + networks.create_column(networks_ra_server) + services.create_column(services_availability_zone) diff --git a/nova/db/sqlalchemy/migrate_repo/versions/__init__.py b/nova/db/sqlalchemy/migrate_repo/versions/__init__.py new file mode 100644 index 000000000..e69de29bb --- /dev/null +++ b/nova/db/sqlalchemy/migrate_repo/versions/__init__.py diff --git a/nova/db/sqlalchemy/migration.py b/nova/db/sqlalchemy/migration.py new file mode 100644 index 000000000..33d14827b --- /dev/null +++ b/nova/db/sqlalchemy/migration.py @@ -0,0 +1,72 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from nova import flags + +import sqlalchemy +from migrate.versioning import api as versioning_api +from migrate.versioning import exceptions as versioning_exceptions + +FLAGS = flags.FLAGS + + +def db_sync(version=None): + db_version() + repo_path = _find_migrate_repo() + return versioning_api.upgrade(FLAGS.sql_connection, repo_path, version) + + +def db_version(): + repo_path = _find_migrate_repo() + try: + return versioning_api.db_version(FLAGS.sql_connection, repo_path) + except versioning_exceptions.DatabaseNotControlledError: + # If we aren't version controlled we may already have the database + # in the state from before we started version control, check for that + # and set up version_control appropriately + meta = sqlalchemy.MetaData() + engine = sqlalchemy.create_engine(FLAGS.sql_connection, echo=False) + meta.reflect(bind=engine) + try: + for table in ('auth_tokens', 'export_devices', 'fixed_ips', + 'floating_ips', 'instances', 'iscsi_targets', + 'key_pairs', 'networks', 'projects', 'quotas', + 'security_group_rules', + 'security_group_instance_association', 'services', + 'users', 'user_project_association', + 'user_project_role_association', 'volumes'): + assert table in meta.tables + return db_version_control(1) + except AssertionError: + return db_version_control(0) + + +def db_version_control(version=None): + repo_path = _find_migrate_repo() + versioning_api.version_control(FLAGS.sql_connection, repo_path, version) + return version + + +def _find_migrate_repo(): + """Get the path for the migrate repository.""" + path = os.path.join(os.path.abspath(os.path.dirname(__file__)), + 'migrate_repo') + assert os.path.exists(path) + return path diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index bf5e48b04..c54ebe3ba 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -100,51 +100,6 @@ class NovaBase(object): return local.iteritems() -# TODO(vish): Store images in the database instead of file system -#class Image(BASE, NovaBase): -# """Represents an image in the datastore""" -# __tablename__ = 'images' -# id = Column(Integer, primary_key=True) -# ec2_id = Column(String(12), unique=True) -# user_id = Column(String(255)) -# project_id = Column(String(255)) -# image_type = Column(String(255)) -# public = Column(Boolean, default=False) -# state = Column(String(255)) -# location = Column(String(255)) -# arch = Column(String(255)) -# default_kernel_id = Column(String(255)) -# default_ramdisk_id = Column(String(255)) -# -# @validates('image_type') -# def validate_image_type(self, key, image_type): -# assert(image_type in ['machine', 'kernel', 'ramdisk', 'raw']) -# -# @validates('state') -# def validate_state(self, key, state): -# assert(state in ['available', 'pending', 'disabled']) -# -# @validates('default_kernel_id') -# def validate_kernel_id(self, key, val): -# if val != 'machine': -# assert(val is None) -# -# @validates('default_ramdisk_id') -# def validate_ramdisk_id(self, key, val): -# if val != 'machine': -# assert(val is None) -# -# -# TODO(vish): To make this into its own table, we need a good place to -# create the host entries. In config somwhere? Or the first -# time any object sets host? This only becomes particularly -# important if we need to store per-host data. -#class Host(BASE, NovaBase): -# """Represents a host where services are running""" -# __tablename__ = 'hosts' -# id = Column(String(255), primary_key=True) - - class Service(BASE, NovaBase): """Represents a running service on a host.""" diff --git a/nova/flags.py b/nova/flags.py index ef66c3f3a..81e2e36f9 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -254,14 +254,15 @@ DEFINE_string('rabbit_virtual_host', '/', 'rabbit virtual host') DEFINE_integer('rabbit_retry_interval', 10, 'rabbit connection retry interval') DEFINE_integer('rabbit_max_retries', 12, 'rabbit connection attempts') DEFINE_string('control_exchange', 'nova', 'the main exchange to connect to') -DEFINE_string('ec2_prefix', 'http', 'prefix for ec2') -DEFINE_string('os_prefix', 'http', 'prefix for openstack') -DEFINE_string('cc_host', '$my_ip', 'ip of api server') -DEFINE_string('cc_dmz', '$my_ip', 'internal ip of api server') -DEFINE_integer('cc_port', 8773, 'cloud controller port') +DEFINE_string('ec2_host', '$my_ip', 'ip of api server') +DEFINE_string('ec2_dmz_host', '$my_ip', 'internal ip of api server') +DEFINE_integer('ec2_port', 8773, 'cloud controller port') +DEFINE_string('ec2_scheme', 'http', 'prefix for ec2') +DEFINE_string('ec2_path', '/services/Cloud', 'suffix for ec2') +DEFINE_string('osapi_host', '$my_ip', 'ip of api server') +DEFINE_string('osapi_scheme', 'http', 'prefix for openstack') DEFINE_integer('osapi_port', 8774, 'OpenStack API port') -DEFINE_string('ec2_suffix', '/services/Cloud', 'suffix for ec2') -DEFINE_string('os_suffix', '/v1.0/', 'suffix for openstack') +DEFINE_string('osapi_path', '/v1.0/', 'suffix for openstack') DEFINE_string('default_project', 'openstack', 'default project for openstack') DEFINE_string('default_image', 'ami-11111', diff --git a/nova/image/glance.py b/nova/image/glance.py index a3a2f4308..593c4bce6 100644 --- a/nova/image/glance.py +++ b/nova/image/glance.py @@ -14,9 +14,9 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - """Implementation of an image service that uses Glance as the backend""" +from __future__ import absolute_import import httplib import json import urlparse @@ -24,171 +24,40 @@ import urlparse from nova import exception from nova import flags from nova import log as logging +from nova import utils from nova.image import service LOG = logging.getLogger('nova.image.glance') FLAGS = flags.FLAGS -flags.DEFINE_string('glance_teller_address', 'http://127.0.0.1', - 'IP address or URL where Glance\'s Teller service resides') -flags.DEFINE_string('glance_teller_port', '9191', - 'Port for Glance\'s Teller service') -flags.DEFINE_string('glance_parallax_address', 'http://127.0.0.1', - 'IP address or URL where Glance\'s Parallax service ' - 'resides') -flags.DEFINE_string('glance_parallax_port', '9292', - 'Port for Glance\'s Parallax service') - - -class TellerClient(object): - - def __init__(self): - self.address = FLAGS.glance_teller_address - self.port = FLAGS.glance_teller_port - url = urlparse.urlparse(self.address) - self.netloc = url.netloc - self.connection_type = {'http': httplib.HTTPConnection, - 'https': httplib.HTTPSConnection}[url.scheme] - -class ParallaxClient(object): - - def __init__(self): - self.address = FLAGS.glance_parallax_address - self.port = FLAGS.glance_parallax_port - url = urlparse.urlparse(self.address) - self.netloc = url.netloc - self.connection_type = {'http': httplib.HTTPConnection, - 'https': httplib.HTTPSConnection}[url.scheme] - - def get_image_index(self): - """ - Returns a list of image id/name mappings from Parallax - """ - try: - c = self.connection_type(self.netloc, self.port) - c.request("GET", "images") - res = c.getresponse() - if res.status == 200: - # Parallax returns a JSONified dict(images=image_list) - data = json.loads(res.read())['images'] - return data - else: - LOG.warn(_("Parallax returned HTTP error %d from " - "request for /images"), res.status_int) - return [] - finally: - c.close() - - def get_image_details(self): - """ - Returns a list of detailed image data mappings from Parallax - """ - try: - c = self.connection_type(self.netloc, self.port) - c.request("GET", "images/detail") - res = c.getresponse() - if res.status == 200: - # Parallax returns a JSONified dict(images=image_list) - data = json.loads(res.read())['images'] - return data - else: - LOG.warn(_("Parallax returned HTTP error %d from " - "request for /images/detail"), res.status_int) - return [] - finally: - c.close() - - def get_image_metadata(self, image_id): - """ - Returns a mapping of image metadata from Parallax - """ - try: - c = self.connection_type(self.netloc, self.port) - c.request("GET", "images/%s" % image_id) - res = c.getresponse() - if res.status == 200: - # Parallax returns a JSONified dict(image=image_info) - data = json.loads(res.read())['image'] - return data - else: - # TODO(jaypipes): log the error? - return None - finally: - c.close() - - def add_image_metadata(self, image_metadata): - """ - Tells parallax about an image's metadata - """ - try: - c = self.connection_type(self.netloc, self.port) - body = json.dumps(image_metadata) - c.request("POST", "images", body) - res = c.getresponse() - if res.status == 200: - # Parallax returns a JSONified dict(image=image_info) - data = json.loads(res.read())['image'] - return data['id'] - else: - # TODO(jaypipes): log the error? - return None - finally: - c.close() - - def update_image_metadata(self, image_id, image_metadata): - """ - Updates Parallax's information about an image - """ - try: - c = self.connection_type(self.netloc, self.port) - body = json.dumps(image_metadata) - c.request("PUT", "images/%s" % image_id, body) - res = c.getresponse() - return res.status == 200 - finally: - c.close() - - def delete_image_metadata(self, image_id): - """ - Deletes Parallax's information about an image - """ - try: - c = self.connection_type(self.netloc, self.port) - c.request("DELETE", "images/%s" % image_id) - res = c.getresponse() - return res.status == 200 - finally: - c.close() +GlanceClient = utils.import_class('glance.client.Client') class GlanceImageService(service.BaseImageService): """Provides storage and retrieval of disk image objects within Glance.""" def __init__(self): - self.teller = TellerClient() - self.parallax = ParallaxClient() + self.client = GlanceClient(FLAGS.glance_host, FLAGS.glance_port) def index(self, context): """ - Calls out to Parallax for a list of images available + Calls out to Glance for a list of images available """ - images = self.parallax.get_image_index() - return images + return self.client.get_images() def detail(self, context): """ - Calls out to Parallax for a list of detailed image information + Calls out to Glance for a list of detailed image information """ - images = self.parallax.get_image_details() - return images + return self.client.get_images_detailed() def show(self, context, id): """ Returns a dict containing image data for the given opaque image id. """ - image = self.parallax.get_image_metadata(id) + image = self.client.get_image_meta(id) if image: return image raise exception.NotFound @@ -200,7 +69,7 @@ class GlanceImageService(service.BaseImageService): :raises AlreadyExists if the image already exist. """ - return self.parallax.add_image_metadata(data) + return self.client.add_image(image_meta=data) def update(self, context, image_id, data): """Replace the contents of the given image with the new data. @@ -208,7 +77,7 @@ class GlanceImageService(service.BaseImageService): :raises NotFound if the image does not exist. """ - self.parallax.update_image_metadata(image_id, data) + return self.client.update_image(image_id, data) def delete(self, context, image_id): """ @@ -217,7 +86,7 @@ class GlanceImageService(service.BaseImageService): :raises NotFound if the image does not exist. """ - self.parallax.delete_image_metadata(image_id) + return self.client.delete_image(image_id) def delete_all(self): """ diff --git a/nova/log.py b/nova/log.py index c1428c051..4997d3f28 100644 --- a/nova/log.py +++ b/nova/log.py @@ -116,6 +116,8 @@ def basicConfig(): handler.setFormatter(_formatter) if FLAGS.verbose: logging.root.setLevel(logging.DEBUG) + else: + logging.root.setLevel(logging.INFO) if FLAGS.use_syslog: syslog = SysLogHandler(address='/dev/log') syslog.setFormatter(_formatter) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 891e9bc1d..d29e17603 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -61,7 +61,7 @@ def metadata_forward(): """Create forwarding rule for metadata""" _confirm_rule("PREROUTING", "-t nat -s 0.0.0.0/0 " "-d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT " - "--to-destination %s:%s" % (FLAGS.cc_dmz, FLAGS.cc_port)) + "--to-destination %s:%s" % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) def init_host(): diff --git a/nova/network/manager.py b/nova/network/manager.py index 2a043cc6b..61de8055a 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -92,7 +92,7 @@ flags.DEFINE_bool('update_dhcp_on_disassociate', False, flags.DEFINE_integer('fixed_ip_disassociate_timeout', 600, 'Seconds after which a deallocated ip is disassociated') -flags.DEFINE_bool('use_ipv6', True, +flags.DEFINE_bool('use_ipv6', False, 'use the ipv6') flags.DEFINE_string('network_host', socket.gethostname(), 'Network host to use for ip allocation in flat modes') diff --git a/nova/service.py b/nova/service.py index 8b2a22ce0..efc08fd63 100644 --- a/nova/service.py +++ b/nova/service.py @@ -209,19 +209,6 @@ class Service(object): self.model_disconnected = True logging.exception(_("model server went away")) - try: - # NOTE(vish): This is late-loaded to make sure that the - # database is not created before flags have - # been loaded. - from nova.db.sqlalchemy import models - models.register_models() - except OperationalError: - logging.exception(_("Data store %s is unreachable." - " Trying again in %d seconds.") % - (FLAGS.sql_connection, - FLAGS.sql_retry_interval)) - time.sleep(FLAGS.sql_retry_interval) - def serve(*services): FLAGS(sys.argv) diff --git a/nova/tests/__init__.py b/nova/tests/__init__.py index 8dc87d0e2..592d5bea9 100644 --- a/nova/tests/__init__.py +++ b/nova/tests/__init__.py @@ -34,3 +34,8 @@ # The code below enables nosetests to work with i18n _() blocks import __builtin__ setattr(__builtin__, '_', lambda x: x) + + +def setup(): + from nova.db import migration + migration.db_sync() diff --git a/nova/tests/api/openstack/__init__.py b/nova/tests/api/openstack/__init__.py index 9e183bd0d..14eaaa62c 100644 --- a/nova/tests/api/openstack/__init__.py +++ b/nova/tests/api/openstack/__init__.py @@ -15,23 +15,28 @@ # License for the specific language governing permissions and limitations # under the License. +import webob.dec import unittest from nova import context from nova import flags from nova.api.openstack.ratelimiting import RateLimitingMiddleware from nova.api.openstack.common import limited -from nova.tests.api.fakes import APIStub -from nova import utils +from nova.tests.api.openstack import fakes from webob import Request FLAGS = flags.FLAGS +@webob.dec.wsgify +def simple_wsgi(req): + return "" + + class RateLimitingMiddlewareTest(unittest.TestCase): def test_get_action_name(self): - middleware = RateLimitingMiddleware(APIStub()) + middleware = RateLimitingMiddleware(simple_wsgi) def verify(method, url, action_name): req = Request.blank(url) @@ -61,19 +66,19 @@ class RateLimitingMiddlewareTest(unittest.TestCase): self.assertTrue('Retry-After' in resp.headers) def test_single_action(self): - middleware = RateLimitingMiddleware(APIStub()) + middleware = RateLimitingMiddleware(simple_wsgi) self.exhaust(middleware, 'DELETE', '/servers/4', 'usr1', 100) self.exhaust(middleware, 'DELETE', '/servers/4', 'usr2', 100) def test_POST_servers_action_implies_POST_action(self): - middleware = RateLimitingMiddleware(APIStub()) + middleware = RateLimitingMiddleware(simple_wsgi) self.exhaust(middleware, 'POST', '/servers/4', 'usr1', 10) self.exhaust(middleware, 'POST', '/images/4', 'usr2', 10) self.assertTrue(set(middleware.limiter._levels) == \ set(['usr1:POST', 'usr1:POST servers', 'usr2:POST'])) def test_POST_servers_action_correctly_ratelimited(self): - middleware = RateLimitingMiddleware(APIStub()) + middleware = RateLimitingMiddleware(simple_wsgi) # Use up all of our "POST" allowance for the minute, 5 times for i in range(5): self.exhaust(middleware, 'POST', '/servers/4', 'usr1', 10) @@ -83,9 +88,9 @@ class RateLimitingMiddlewareTest(unittest.TestCase): self.exhaust(middleware, 'POST', '/servers/4', 'usr1', 0) def test_proxy_ctor_works(self): - middleware = RateLimitingMiddleware(APIStub()) + middleware = RateLimitingMiddleware(simple_wsgi) self.assertEqual(middleware.limiter.__class__.__name__, "Limiter") - middleware = RateLimitingMiddleware(APIStub(), service_host='foobar') + middleware = RateLimitingMiddleware(simple_wsgi, service_host='foobar') self.assertEqual(middleware.limiter.__class__.__name__, "WSGIAppProxy") diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 194304e79..fb282f1c9 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -22,6 +22,9 @@ import string import webob import webob.dec +from paste import urlmap + +from glance import client as glance_client from nova import auth from nova import context @@ -29,6 +32,7 @@ from nova import exception as exc from nova import flags from nova import utils import nova.api.openstack.auth +from nova.api import openstack from nova.api.openstack import auth from nova.api.openstack import ratelimiting from nova.image import glance @@ -69,6 +73,17 @@ def fake_wsgi(self, req): return self.application +def wsgi_app(inner_application=None): + if not inner_application: + inner_application = openstack.APIRouter() + mapper = urlmap.URLMap() + api = openstack.FaultWrapper(auth.AuthMiddleware( + ratelimiting.RateLimitingMiddleware(inner_application))) + mapper['/v1.0'] = api + mapper['/'] = openstack.FaultWrapper(openstack.Versions()) + return mapper + + def stub_out_key_pair_funcs(stubs): def key_pair(context, user_id): return [dict(name='key', public_key='public_key')] @@ -116,64 +131,60 @@ def stub_out_compute_api_snapshot(stubs): stubs.Set(nova.compute.API, 'snapshot', snapshot) -def stub_out_glance(stubs, initial_fixtures=[]): +def stub_out_glance(stubs, initial_fixtures=None): - class FakeParallaxClient: + class FakeGlanceClient: def __init__(self, initial_fixtures): - self.fixtures = initial_fixtures + self.fixtures = initial_fixtures or [] - def fake_get_image_index(self): + def fake_get_images(self): return [dict(id=f['id'], name=f['name']) for f in self.fixtures] - def fake_get_image_details(self): + def fake_get_images_detailed(self): return self.fixtures - def fake_get_image_metadata(self, image_id): + def fake_get_image_meta(self, image_id): for f in self.fixtures: if f['id'] == image_id: return f return None - def fake_add_image_metadata(self, image_data): + def fake_add_image(self, image_meta): id = ''.join(random.choice(string.letters) for _ in range(20)) - image_data['id'] = id - self.fixtures.append(image_data) + image_meta['id'] = id + self.fixtures.append(image_meta) return id - def fake_update_image_metadata(self, image_id, image_data): - f = self.fake_get_image_metadata(image_id) + def fake_update_image(self, image_id, image_meta): + f = self.fake_get_image_meta(image_id) if not f: raise exc.NotFound - f.update(image_data) + f.update(image_meta) - def fake_delete_image_metadata(self, image_id): - f = self.fake_get_image_metadata(image_id) + def fake_delete_image(self, image_id): + f = self.fake_get_image_meta(image_id) if not f: raise exc.NotFound self.fixtures.remove(f) - def fake_delete_all(self): - self.fixtures = [] - - fake_parallax_client = FakeParallaxClient(initial_fixtures) - stubs.Set(nova.image.glance.ParallaxClient, 'get_image_index', - fake_parallax_client.fake_get_image_index) - stubs.Set(nova.image.glance.ParallaxClient, 'get_image_details', - fake_parallax_client.fake_get_image_details) - stubs.Set(nova.image.glance.ParallaxClient, 'get_image_metadata', - fake_parallax_client.fake_get_image_metadata) - stubs.Set(nova.image.glance.ParallaxClient, 'add_image_metadata', - fake_parallax_client.fake_add_image_metadata) - stubs.Set(nova.image.glance.ParallaxClient, 'update_image_metadata', - fake_parallax_client.fake_update_image_metadata) - stubs.Set(nova.image.glance.ParallaxClient, 'delete_image_metadata', - fake_parallax_client.fake_delete_image_metadata) - stubs.Set(nova.image.glance.GlanceImageService, 'delete_all', - fake_parallax_client.fake_delete_all) + ##def fake_delete_all(self): + ## self.fixtures = [] + + GlanceClient = glance_client.Client + fake = FakeGlanceClient(initial_fixtures) + + stubs.Set(GlanceClient, 'get_images', fake.fake_get_images) + stubs.Set(GlanceClient, 'get_images_detailed', + fake.fake_get_images_detailed) + stubs.Set(GlanceClient, 'get_image_meta', fake.fake_get_image_meta) + stubs.Set(GlanceClient, 'add_image', fake.fake_add_image) + stubs.Set(GlanceClient, 'update_image', fake.fake_update_image) + stubs.Set(GlanceClient, 'delete_image', fake.fake_delete_image) + #stubs.Set(GlanceClient, 'delete_all', fake.fake_delete_all) class FakeToken(object): diff --git a/nova/tests/api/openstack/test_adminapi.py b/nova/tests/api/openstack/test_adminapi.py index 1b2e1654d..73120c31d 100644 --- a/nova/tests/api/openstack/test_adminapi.py +++ b/nova/tests/api/openstack/test_adminapi.py @@ -19,15 +19,19 @@ import unittest import stubout import webob +from paste import urlmap -import nova.api from nova import flags +from nova.api import openstack +from nova.api.openstack import ratelimiting +from nova.api.openstack import auth from nova.tests.api.openstack import fakes FLAGS = flags.FLAGS class AdminAPITest(unittest.TestCase): + def setUp(self): self.stubs = stubout.StubOutForTesting() fakes.FakeAuthManager.auth_data = {} @@ -45,7 +49,7 @@ class AdminAPITest(unittest.TestCase): FLAGS.allow_admin_api = True # We should still be able to access public operations. req = webob.Request.blank('/v1.0/flavors') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) # TODO: Confirm admin operations are available. @@ -53,7 +57,7 @@ class AdminAPITest(unittest.TestCase): FLAGS.allow_admin_api = False # We should still be able to access public operations. req = webob.Request.blank('/v1.0/flavors') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) # TODO: Confirm admin operations are unavailable. diff --git a/nova/tests/api/openstack/test_api.py b/nova/tests/api/openstack/test_api.py index d8b202e21..db0fe1060 100644 --- a/nova/tests/api/openstack/test_api.py +++ b/nova/tests/api/openstack/test_api.py @@ -19,14 +19,18 @@ import unittest import webob.exc import webob.dec -import nova.api.openstack -from nova.api.openstack import API -from nova.api.openstack import faults from webob import Request +from nova.api import openstack +from nova.api.openstack import faults + class APITest(unittest.TestCase): + def _wsgi_app(self, inner_app): + # simpler version of the app than fakes.wsgi_app + return openstack.FaultWrapper(inner_app) + def test_exceptions_are_converted_to_faults(self): @webob.dec.wsgify @@ -46,29 +50,32 @@ class APITest(unittest.TestCase): exc = webob.exc.HTTPNotFound(explanation='Raised a webob.exc') return faults.Fault(exc) - api = API() - - api.application = succeed + #api.application = succeed + api = self._wsgi_app(succeed) resp = Request.blank('/').get_response(api) self.assertFalse('computeFault' in resp.body, resp.body) self.assertEqual(resp.status_int, 200, resp.body) - api.application = raise_webob_exc + #api.application = raise_webob_exc + api = self._wsgi_app(raise_webob_exc) resp = Request.blank('/').get_response(api) self.assertFalse('computeFault' in resp.body, resp.body) self.assertEqual(resp.status_int, 404, resp.body) - api.application = raise_api_fault + #api.application = raise_api_fault + api = self._wsgi_app(raise_api_fault) resp = Request.blank('/').get_response(api) self.assertTrue('itemNotFound' in resp.body, resp.body) self.assertEqual(resp.status_int, 404, resp.body) - api.application = fail + #api.application = fail + api = self._wsgi_app(fail) resp = Request.blank('/').get_response(api) self.assertTrue('{"computeFault' in resp.body, resp.body) self.assertEqual(resp.status_int, 500, resp.body) - api.application = fail + #api.application = fail + api = self._wsgi_app(fail) resp = Request.blank('/.xml').get_response(api) self.assertTrue('<computeFault' in resp.body, resp.body) self.assertEqual(resp.status_int, 500, resp.body) diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 489a1dfbf..0dd65d321 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -53,7 +53,7 @@ class Test(unittest.TestCase): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) self.assertEqual(result.headers['X-CDN-Management-Url'], @@ -67,7 +67,7 @@ class Test(unittest.TestCase): req = webob.Request.blank('/v1.0/', {'HTTP_HOST': 'foo'}) req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '204 No Content') self.assertEqual(len(result.headers['X-Auth-Token']), 40) self.assertEqual(result.headers['X-Server-Management-Url'], @@ -81,7 +81,7 @@ class Test(unittest.TestCase): fakes.FakeRouter) req = webob.Request.blank('/v1.0/fake') req.headers['X-Auth-Token'] = token - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '200 OK') self.assertEqual(result.headers['X-Test-Success'], 'True') @@ -105,7 +105,7 @@ class Test(unittest.TestCase): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-Token'] = 'bacon' - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') self.assertEqual(self.destroy_called, True) @@ -113,18 +113,18 @@ class Test(unittest.TestCase): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') def test_no_user(self): req = webob.Request.blank('/v1.0/') - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') def test_bad_token(self): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-Token'] = 'baconbaconbacon' - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '401 Unauthorized') @@ -149,7 +149,7 @@ class TestLimiter(unittest.TestCase): req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(len(result.headers['X-Auth-Token']), 40) token = result.headers['X-Auth-Token'] @@ -158,7 +158,7 @@ class TestLimiter(unittest.TestCase): req = webob.Request.blank('/v1.0/fake') req.method = 'POST' req.headers['X-Auth-Token'] = token - result = req.get_response(nova.api.API('os')) + result = req.get_response(fakes.wsgi_app()) self.assertEqual(result.status, '200 OK') self.assertEqual(result.headers['X-Test-Success'], 'True') diff --git a/nova/tests/api/openstack/test_flavors.py b/nova/tests/api/openstack/test_flavors.py index 41018afdf..1bdaea161 100644 --- a/nova/tests/api/openstack/test_flavors.py +++ b/nova/tests/api/openstack/test_flavors.py @@ -39,7 +39,7 @@ class FlavorsTest(unittest.TestCase): def test_get_flavor_list(self): req = webob.Request.blank('/v1.0/flavors') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) def test_get_flavor_by_id(self): pass diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index 00ca739a5..5d9ddefbe 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -210,7 +210,7 @@ class ImageControllerWithGlanceServiceTest(unittest.TestCase): def test_get_image_index(self): req = webob.Request.blank('/v1.0/images') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) fixture_index = [dict(id=f['id'], name=f['name']) for f @@ -222,7 +222,7 @@ class ImageControllerWithGlanceServiceTest(unittest.TestCase): def test_get_image_details(self): req = webob.Request.blank('/v1.0/images/detail') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) def _is_equivalent_subset(x, y): diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 0396daf98..29883e7c8 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -100,14 +100,14 @@ class ServersTest(unittest.TestCase): def test_get_server_by_id(self): req = webob.Request.blank('/v1.0/servers/1') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) self.assertEqual(res_dict['server']['id'], '1') self.assertEqual(res_dict['server']['name'], 'server1') def test_get_server_list(self): req = webob.Request.blank('/v1.0/servers') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) i = 0 @@ -160,14 +160,14 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) def test_update_no_body(self): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 422) def test_update_bad_params(self): @@ -186,7 +186,7 @@ class ServersTest(unittest.TestCase): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' req.body = self.body - req.get_response(nova.api.API('os')) + req.get_response(fakes.wsgi_app()) def test_update_server(self): inst_dict = dict(name='server_test', adminPass='bacon') @@ -202,28 +202,28 @@ class ServersTest(unittest.TestCase): req = webob.Request.blank('/v1.0/servers/1') req.method = 'PUT' req.body = self.body - req.get_response(nova.api.API('os')) + req.get_response(fakes.wsgi_app()) def test_create_backup_schedules(self): req = webob.Request.blank('/v1.0/servers/1/backup_schedules') req.method = 'POST' - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_delete_backup_schedules(self): req = webob.Request.blank('/v1.0/servers/1/backup_schedules') req.method = 'DELETE' - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_get_server_backup_schedules(self): req = webob.Request.blank('/v1.0/servers/1/backup_schedules') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '404 Not Found') def test_get_all_server_details(self): req = webob.Request.blank('/v1.0/servers/detail') - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) res_dict = json.loads(res.body) i = 0 @@ -242,7 +242,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 202) def test_server_unpause(self): @@ -254,7 +254,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 202) def test_server_suspend(self): @@ -266,7 +266,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 202) def test_server_resume(self): @@ -278,19 +278,19 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 202) def test_server_diagnostics(self): req = webob.Request.blank("/v1.0/servers/1/diagnostics") req.method = "GET" - res = req.get_response(nova.api.API("os")) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 404) def test_server_actions(self): req = webob.Request.blank("/v1.0/servers/1/actions") req.method = "GET" - res = req.get_response(nova.api.API("os")) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 404) def test_server_reboot(self): @@ -301,7 +301,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) def test_server_rebuild(self): body = dict(server=dict( @@ -311,7 +311,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) def test_server_resize(self): body = dict(server=dict( @@ -321,7 +321,7 @@ class ServersTest(unittest.TestCase): req.method = 'POST' req.content_type = 'application/json' req.body = json.dumps(body) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) def test_delete_server_instance(self): req = webob.Request.blank('/v1.0/servers/1') @@ -335,7 +335,7 @@ class ServersTest(unittest.TestCase): self.stubs.Set(nova.db.api, 'instance_destroy', instance_destroy_mock) - res = req.get_response(nova.api.API('os')) + res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status, '202 Accepted') self.assertEqual(self.server_delete_called, True) diff --git a/nova/tests/api/test.py b/nova/tests/api/test.py deleted file mode 100644 index 9caa8c9d0..000000000 --- a/nova/tests/api/test.py +++ /dev/null @@ -1,81 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -""" -Test for the root WSGI middleware for all API controllers. -""" - -import unittest - -import stubout -import webob -import webob.dec - -import nova.exception -from nova import api -from nova.tests.api.fakes import APIStub - - -class Test(unittest.TestCase): - - def setUp(self): - self.stubs = stubout.StubOutForTesting() - - def tearDown(self): - self.stubs.UnsetAll() - - def _request(self, url, subdomain, **kwargs): - environ_keys = {'HTTP_HOST': '%s.example.com' % subdomain} - environ_keys.update(kwargs) - req = webob.Request.blank(url, environ_keys) - return req.get_response(api.API('ec2')) - - def test_openstack(self): - self.stubs.Set(api.openstack, 'API', APIStub) - result = self._request('/v1.0/cloud', 'api') - self.assertEqual(result.body, "/cloud") - - def test_ec2(self): - self.stubs.Set(api.ec2, 'API', APIStub) - result = self._request('/services/cloud', 'ec2') - self.assertEqual(result.body, "/cloud") - - def test_not_found(self): - self.stubs.Set(api.ec2, 'API', APIStub) - self.stubs.Set(api.openstack, 'API', APIStub) - result = self._request('/test/cloud', 'ec2') - self.assertNotEqual(result.body, "/cloud") - - def test_query_api_versions(self): - result = self._request('/', 'api') - self.assertTrue('CURRENT' in result.body) - - def test_metadata(self): - def go(url): - result = self._request(url, 'ec2', REMOTE_ADDR='128.192.151.2') - # Each should get to the ORM layer and fail to find the IP - self.assertRaises(nova.exception.NotFound, go, '/latest/') - self.assertRaises(nova.exception.NotFound, go, '/2009-04-04/') - self.assertRaises(nova.exception.NotFound, go, '/1.0/') - - def test_ec2_root(self): - result = self._request('/', 'ec2') - self.assertTrue('2007-12-15\n' in result.body) - - -if __name__ == '__main__': - unittest.main() diff --git a/nova/tests/fake_flags.py b/nova/tests/fake_flags.py index 7376a11dd..1097488ec 100644 --- a/nova/tests/fake_flags.py +++ b/nova/tests/fake_flags.py @@ -40,3 +40,4 @@ FLAGS.blades_per_shelf = 4 FLAGS.iscsi_num_targets = 8 FLAGS.verbose = True FLAGS.sql_connection = 'sqlite:///nova.sqlite' +FLAGS.use_ipv6 = True diff --git a/nova/tests/api/fakes.py b/nova/tests/glance/__init__.py index 0aedcaff0..ef9fa05a7 100644 --- a/nova/tests/api/fakes.py +++ b/nova/tests/glance/__init__.py @@ -1,7 +1,6 @@ # vim: tabstop=4 shiftwidth=4 softtabstop=4 -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. +# Copyright (c) 2011 Citrix Systems, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -15,12 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -import webob.dec -from nova import wsgi - - -class APIStub(object): - """Class to verify request and mark it was called.""" - @webob.dec.wsgify - def __call__(self, req): - return req.path_info +""" +:mod:`glance` -- Stubs for Glance +================================= +""" diff --git a/nova/tests/glance/stubs.py b/nova/tests/glance/stubs.py new file mode 100644 index 000000000..f182b857a --- /dev/null +++ b/nova/tests/glance/stubs.py @@ -0,0 +1,37 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 Citrix Systems, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import StringIO + +import glance.client + + +def stubout_glance_client(stubs, cls): + """Stubs out glance.client.Client""" + stubs.Set(glance.client, 'Client', + lambda *args, **kwargs: cls(*args, **kwargs)) + + +class FakeGlance(object): + def __init__(self, host, port=None, use_ssl=False): + pass + + def get_image(self, image): + meta = { + 'size': 0, + } + image_file = StringIO.StringIO('') + return meta, image_file diff --git a/nova/tests/test_access.py b/nova/tests/test_access.py index 0929903cf..e170ccee6 100644 --- a/nova/tests/test_access.py +++ b/nova/tests/test_access.py @@ -20,21 +20,31 @@ import unittest import webob from nova import context -from nova import exception from nova import flags from nova import test from nova.api import ec2 from nova.auth import manager - FLAGS = flags.FLAGS -class Context(object): +class FakeControllerClass(object): pass +class FakeApiRequest(object): + def __init__(self, action): + self.controller = FakeControllerClass() + self.action = action + + class AccessTestCase(test.TestCase): + def _env_for(self, ctxt, action): + env = {} + env['ec2.context'] = ctxt + env['ec2.request'] = FakeApiRequest(action) + return env + def setUp(self): super(AccessTestCase, self).setUp() um = manager.AuthManager() @@ -64,7 +74,7 @@ class AccessTestCase(test.TestCase): return [''] self.mw = ec2.Authorizer(noopWSGIApp) - self.mw.action_roles = {'str': { + self.mw.action_roles = {'FakeControllerClass': { '_allow_all': ['all'], '_allow_none': [], '_allow_project_manager': ['projectmanager'], @@ -84,9 +94,7 @@ class AccessTestCase(test.TestCase): def response_status(self, user, methodName): ctxt = context.RequestContext(user, self.project) - environ = {'ec2.context': ctxt, - 'ec2.controller': 'some string', - 'ec2.action': methodName} + environ = self._env_for(ctxt, methodName) req = webob.Request.blank('/', environ) resp = req.get_response(self.mw) return resp.status_int diff --git a/nova/tests/test_api.py b/nova/tests/test_api.py index 17789c25c..66a16b0cb 100644 --- a/nova/tests/test_api.py +++ b/nova/tests/test_api.py @@ -26,9 +26,8 @@ import StringIO import webob from nova import context -from nova import flags from nova import test -from nova import api +from nova.api import ec2 from nova.api.ec2 import cloud from nova.api.ec2 import apirequest from nova.auth import manager @@ -100,12 +99,10 @@ class ApiEc2TestCase(test.TestCase): """Unit test for the cloud controller on an EC2 API""" def setUp(self): super(ApiEc2TestCase, self).setUp() - self.manager = manager.AuthManager() - self.host = '127.0.0.1' - - self.app = api.API('ec2') + self.app = ec2.Authenticate(ec2.Requestify(ec2.Executor(), + 'nova.api.ec2.cloud.CloudController')) def expect_http(self, host=None, is_secure=False): """Returns a new EC2 connection""" diff --git a/nova/tests/test_compute.py b/nova/tests/test_compute.py index f50f5755f..09f6ee94a 100644 --- a/nova/tests/test_compute.py +++ b/nova/tests/test_compute.py @@ -32,8 +32,9 @@ from nova import utils from nova.auth import manager -FLAGS = flags.FLAGS LOG = logging.getLogger('nova.tests.compute') +FLAGS = flags.FLAGS +flags.DECLARE('stub_network', 'nova.compute.manager') class ComputeTestCase(test.TestCase): diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index afdc89ba2..f6800e3d9 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -122,10 +122,10 @@ class LibvirtConnTestCase(test.TestCase): if rescue: check = (lambda t: t.find('./os/kernel').text.split('/')[1], - 'rescue-kernel') + 'kernel.rescue') check_list.append(check) check = (lambda t: t.find('./os/initrd').text.split('/')[1], - 'rescue-ramdisk') + 'ramdisk.rescue') check_list.append(check) else: if expect_kernel: @@ -161,13 +161,16 @@ class LibvirtConnTestCase(test.TestCase): if rescue: common_checks += [ (lambda t: t.findall('./devices/disk/source')[0].get( - 'file').split('/')[1], 'rescue-disk'), + 'file').split('/')[1], 'disk.rescue'), (lambda t: t.findall('./devices/disk/source')[1].get( 'file').split('/')[1], 'disk')] else: common_checks += [(lambda t: t.findall( './devices/disk/source')[0].get('file').split('/')[1], 'disk')] + common_checks += [(lambda t: t.findall( + './devices/disk/source')[1].get('file').split('/')[1], + 'disk.local')] for (libvirt_type, (expected_uri, checks)) in type_uri_map.iteritems(): FLAGS.libvirt_type = libvirt_type @@ -225,12 +228,6 @@ class IptablesFirewallTestCase(test.TestCase): self.manager.delete_user(self.user) super(IptablesFirewallTestCase, self).tearDown() - def _p(self, *args, **kwargs): - if 'iptables-restore' in args: - print ' '.join(args), kwargs['stdin'] - if 'iptables-save' in args: - return - in_rules = [ '# Generated by iptables-save v1.4.4 on Mon Dec 6 11:54:13 2010', '*filter', @@ -252,11 +249,21 @@ class IptablesFirewallTestCase(test.TestCase): '# Completed on Mon Dec 6 11:54:13 2010', ] + in6_rules = [ + '# Generated by ip6tables-save v1.4.4 on Tue Jan 18 23:47:56 2011', + '*filter', + ':INPUT ACCEPT [349155:75810423]', + ':FORWARD ACCEPT [0:0]', + ':OUTPUT ACCEPT [349256:75777230]', + 'COMMIT', + '# Completed on Tue Jan 18 23:47:56 2011' + ] + def test_static_filters(self): - self.fw.execute = self._p instance_ref = db.instance_create(self.context, {'user_id': 'fake', - 'project_id': 'fake'}) + 'project_id': 'fake', + 'mac_address': '56:12:12:12:12:12'}) ip = '10.11.12.13' network_ref = db.project_get_network(self.context, @@ -301,18 +308,31 @@ class IptablesFirewallTestCase(test.TestCase): secgroup['id']) instance_ref = db.instance_get(admin_ctxt, instance_ref['id']) - self.fw.add_instance(instance_ref) - - out_rules = self.fw.modify_rules(self.in_rules) +# self.fw.add_instance(instance_ref) + def fake_iptables_execute(cmd, process_input=None): + if cmd == 'sudo ip6tables-save -t filter': + return '\n'.join(self.in6_rules), None + if cmd == 'sudo iptables-save -t filter': + return '\n'.join(self.in_rules), None + if cmd == 'sudo iptables-restore': + self.out_rules = process_input.split('\n') + return '', '' + if cmd == 'sudo ip6tables-restore': + self.out6_rules = process_input.split('\n') + return '', '' + self.fw.execute = fake_iptables_execute + + self.fw.prepare_instance_filter(instance_ref) + self.fw.apply_instance_filter(instance_ref) in_rules = filter(lambda l: not l.startswith('#'), self.in_rules) for rule in in_rules: if not 'nova' in rule: - self.assertTrue(rule in out_rules, + self.assertTrue(rule in self.out_rules, 'Rule went missing: %s' % rule) instance_chain = None - for rule in out_rules: + for rule in self.out_rules: # This is pretty crude, but it'll do for now if '-d 10.11.12.13 -j' in rule: instance_chain = rule.split(' ')[-1] @@ -320,7 +340,7 @@ class IptablesFirewallTestCase(test.TestCase): self.assertTrue(instance_chain, "The instance chain wasn't added") security_group_chain = None - for rule in out_rules: + for rule in self.out_rules: # This is pretty crude, but it'll do for now if '-A %s -j' % instance_chain in rule: security_group_chain = rule.split(' ')[-1] @@ -329,16 +349,16 @@ class IptablesFirewallTestCase(test.TestCase): "The security group chain wasn't added") self.assertTrue('-A %s -p icmp -s 192.168.11.0/24 -j ACCEPT' % \ - security_group_chain in out_rules, + security_group_chain in self.out_rules, "ICMP acceptance rule wasn't added") - self.assertTrue('-A %s -p icmp -s 192.168.11.0/24 -m icmp --icmp-type' - ' 8 -j ACCEPT' % security_group_chain in out_rules, + self.assertTrue('-A %s -p icmp -s 192.168.11.0/24 -m icmp --icmp-type ' + '8 -j ACCEPT' % security_group_chain in self.out_rules, "ICMP Echo Request acceptance rule wasn't added") self.assertTrue('-A %s -p tcp -s 192.168.10.0/24 -m multiport ' '--dports 80:81 -j ACCEPT' % security_group_chain \ - in out_rules, + in self.out_rules, "TCP port 80/81 acceptance rule wasn't added") @@ -473,5 +493,6 @@ class NWFilterTestCase(test.TestCase): self.fw.setup_basic_filtering(instance) self.fw.prepare_instance_filter(instance) + self.fw.apply_instance_filter(instance) _ensure_all_called() self.teardown_security_group() diff --git a/nova/tests/test_xenapi.py b/nova/tests/test_xenapi.py index 261ee0fde..9f5b266f3 100644 --- a/nova/tests/test_xenapi.py +++ b/nova/tests/test_xenapi.py @@ -34,6 +34,7 @@ from nova.virt.xenapi import volume_utils from nova.virt.xenapi.vmops import SimpleDH from nova.tests.db import fakes as db_fakes from nova.tests.xenapi import stubs +from nova.tests.glance import stubs as glance_stubs FLAGS = flags.FLAGS @@ -108,18 +109,16 @@ class XenAPIVolumeTestCase(test.TestCase): conn = xenapi_conn.get_connection(False) volume = self._create_volume() instance = db.instance_create(self.values) - xenapi_fake.create_vm(instance.name, 'Running') + vm = xenapi_fake.create_vm(instance.name, 'Running') result = conn.attach_volume(instance.name, volume['id'], '/dev/sdc') def check(): # check that the VM has a VBD attached to it - # Get XenAPI reference for the VM - vms = xenapi_fake.get_all('VM') # Get XenAPI record for VBD vbds = xenapi_fake.get_all('VBD') vbd = xenapi_fake.get_record('VBD', vbds[0]) vm_ref = vbd['VM'] - self.assertEqual(vm_ref, vms[0]) + self.assertEqual(vm_ref, vm) check() @@ -157,9 +156,14 @@ class XenAPIVMTestCase(test.TestCase): FLAGS.xenapi_connection_url = 'test_url' FLAGS.xenapi_connection_password = 'test_pass' xenapi_fake.reset() + xenapi_fake.create_local_srs() db_fakes.stub_out_db_instance_api(self.stubs) xenapi_fake.create_network('fake', FLAGS.flat_network_bridge) stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) + stubs.stubout_get_this_vm_uuid(self.stubs) + stubs.stubout_stream_disk(self.stubs) + glance_stubs.stubout_glance_client(self.stubs, + glance_stubs.FakeGlance) self.conn = xenapi_conn.get_connection(False) def test_list_instances_0(self): @@ -207,40 +211,70 @@ class XenAPIVMTestCase(test.TestCase): check() - def test_spawn(self): - instance = self._create_instance() + def check_vm_record(self, conn): + instances = conn.list_instances() + self.assertEquals(instances, [1]) + + # Get Nova record for VM + vm_info = conn.get_info(1) + + # Get XenAPI record for VM + vms = [rec for ref, rec + in xenapi_fake.get_all_records('VM').iteritems() + if not rec['is_control_domain']] + vm = vms[0] + + # Check that m1.large above turned into the right thing. + instance_type = instance_types.INSTANCE_TYPES['m1.large'] + mem_kib = long(instance_type['memory_mb']) << 10 + mem_bytes = str(mem_kib << 10) + vcpus = instance_type['vcpus'] + self.assertEquals(vm_info['max_mem'], mem_kib) + self.assertEquals(vm_info['mem'], mem_kib) + self.assertEquals(vm['memory_static_max'], mem_bytes) + self.assertEquals(vm['memory_dynamic_max'], mem_bytes) + self.assertEquals(vm['memory_dynamic_min'], mem_bytes) + self.assertEquals(vm['VCPUs_max'], str(vcpus)) + self.assertEquals(vm['VCPUs_at_startup'], str(vcpus)) + + # Check that the VM is running according to Nova + self.assertEquals(vm_info['state'], power_state.RUNNING) + + # Check that the VM is running according to XenAPI. + self.assertEquals(vm['power_state'], 'Running') + + def _test_spawn(self, image_id, kernel_id, ramdisk_id): + stubs.stubout_session(self.stubs, stubs.FakeSessionForVMTests) + values = {'name': 1, + 'id': 1, + 'project_id': self.project.id, + 'user_id': self.user.id, + 'image_id': image_id, + 'kernel_id': kernel_id, + 'ramdisk_id': ramdisk_id, + 'instance_type': 'm1.large', + 'mac_address': 'aa:bb:cc:dd:ee:ff', + } + conn = xenapi_conn.get_connection(False) + instance = db.instance_create(values) + conn.spawn(instance) + self.check_vm_record(conn) - def check(): - instances = self.conn.list_instances() - self.assertEquals(instances, [1]) - - # Get Nova record for VM - vm_info = self.conn.get_info(1) - - # Get XenAPI record for VM - vms = xenapi_fake.get_all('VM') - vm = xenapi_fake.get_record('VM', vms[0]) - - # Check that m1.large above turned into the right thing. - instance_type = instance_types.INSTANCE_TYPES['m1.large'] - mem_kib = long(instance_type['memory_mb']) << 10 - mem_bytes = str(mem_kib << 10) - vcpus = instance_type['vcpus'] - self.assertEquals(vm_info['max_mem'], mem_kib) - self.assertEquals(vm_info['mem'], mem_kib) - self.assertEquals(vm['memory_static_max'], mem_bytes) - self.assertEquals(vm['memory_dynamic_max'], mem_bytes) - self.assertEquals(vm['memory_dynamic_min'], mem_bytes) - self.assertEquals(vm['VCPUs_max'], str(vcpus)) - self.assertEquals(vm['VCPUs_at_startup'], str(vcpus)) - - # Check that the VM is running according to Nova - self.assertEquals(vm_info['state'], power_state.RUNNING) - - # Check that the VM is running according to XenAPI. - self.assertEquals(vm['power_state'], 'Running') + def test_spawn_raw_objectstore(self): + FLAGS.xenapi_image_service = 'objectstore' + self._test_spawn(1, None, None) - check() + def test_spawn_objectstore(self): + FLAGS.xenapi_image_service = 'objectstore' + self._test_spawn(1, 2, 3) + + def test_spawn_raw_glance(self): + FLAGS.xenapi_image_service = 'glance' + self._test_spawn(1, None, None) + + def test_spawn_glance(self): + FLAGS.xenapi_image_service = 'glance' + self._test_spawn(1, 2, 3) def tearDown(self): super(XenAPIVMTestCase, self).tearDown() diff --git a/nova/tests/xenapi/stubs.py b/nova/tests/xenapi/stubs.py index 292bd9ba9..624995ada 100644 --- a/nova/tests/xenapi/stubs.py +++ b/nova/tests/xenapi/stubs.py @@ -115,6 +115,21 @@ def stub_out_get_target(stubs): stubs.Set(volume_utils, '_get_target', fake_get_target) +def stubout_get_this_vm_uuid(stubs): + def f(): + vms = [rec['uuid'] for ref, rec + in fake.get_all_records('VM').iteritems() + if rec['is_control_domain']] + return vms[0] + stubs.Set(vm_utils, 'get_this_vm_uuid', f) + + +def stubout_stream_disk(stubs): + def f(_1, _2, _3, _4): + pass + stubs.Set(vm_utils, '_stream_disk', f) + + class FakeSessionForVMTests(fake.SessionBase): """ Stubs out a XenAPISession for VM tests """ def __init__(self, uri): @@ -124,7 +139,10 @@ class FakeSessionForVMTests(fake.SessionBase): return self.xenapi.network.get_all_records() def host_call_plugin(self, _1, _2, _3, _4, _5): - return '' + sr_ref = fake.get_all('SR')[0] + vdi_ref = fake.create_vdi('', False, sr_ref, False) + vdi_rec = fake.get_record('VDI', vdi_ref) + return '<string>%s</string>' % vdi_rec['uuid'] def VM_start(self, _1, ref, _2, _3): vm = fake.get_record('VM', ref) @@ -159,10 +177,6 @@ class FakeSessionForVolumeTests(fake.SessionBase): def __init__(self, uri): super(FakeSessionForVolumeTests, self).__init__(uri) - def VBD_plug(self, _1, ref): - rec = fake.get_record('VBD', ref) - rec['currently-attached'] = True - def VDI_introduce(self, _1, uuid, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11): valid_vdi = False diff --git a/nova/utils.py b/nova/utils.py index fdbe81c0c..6d3ddd092 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -334,6 +334,20 @@ class LazyPluggable(object): return getattr(backend, key) +class LoopingCallDone(Exception): + """The poll-function passed to LoopingCall can raise this exception to + break out of the loop normally. This is somewhat analogous to + StopIteration. + + An optional return-value can be included as the argument to the exception; + this return-value will be returned by LoopingCall.wait() + """ + + def __init__(self, retvalue=True): + """:param retvalue: Value that LoopingCall.wait() should return""" + self.retvalue = retvalue + + class LoopingCall(object): def __init__(self, f=None, *args, **kw): self.args = args @@ -352,12 +366,15 @@ class LoopingCall(object): while self._running: self.f(*self.args, **self.kw) greenthread.sleep(interval) + except LoopingCallDone, e: + self.stop() + done.send(e.retvalue) except Exception: logging.exception('in looping call') done.send_exception(*sys.exc_info()) return - - done.send(True) + else: + done.send(True) self.done = done diff --git a/nova/virt/fake.py b/nova/virt/fake.py index a6fe24c3e..ccd937fd5 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -322,6 +322,54 @@ class FakeConnection(object): 'username': 'fakeuser', 'password': 'fakepassword'} + def refresh_security_group_rules(self, security_group_id): + """This method is called after a change to security groups. + + All security groups and their associated rules live in the datastore, + and calling this method should apply the updated rules to instances + running the specified security group. + + An error should be raised if the operation cannot complete. + + """ + return True + + def refresh_security_group_members(self, security_group_id): + """This method is called when a security group is added to an instance. + + This message is sent to the virtualization drivers on hosts that are + running an instance that belongs to a security group that has a rule + that references the security group identified by `security_group_id`. + It is the responsiblity of this method to make sure any rules + that authorize traffic flow with members of the security group are + updated and any new members can communicate, and any removed members + cannot. + + Scenario: + * we are running on host 'H0' and we have an instance 'i-0'. + * instance 'i-0' is a member of security group 'speaks-b' + * group 'speaks-b' has an ingress rule that authorizes group 'b' + * another host 'H1' runs an instance 'i-1' + * instance 'i-1' is a member of security group 'b' + + When 'i-1' launches or terminates we will recieve the message + to update members of group 'b', at which time we will make + any changes needed to the rules for instance 'i-0' to allow + or deny traffic coming from 'i-1', depending on if it is being + added or removed from the group. + + In this scenario, 'i-1' could just as easily have been running on our + host 'H0' and this method would still have been called. The point was + that this method isn't called on the host where instances of that + group are running (as is the case with + :method:`refresh_security_group_rules`) but is called where references + are made to authorizing those instances. + + An error should be raised if the operation cannot complete. + + """ + return True + class FakeInstance(object): diff --git a/nova/virt/libvirt.xml.template b/nova/virt/libvirt.xml.template index de06a1eb0..8139c3620 100644 --- a/nova/virt/libvirt.xml.template +++ b/nova/virt/libvirt.xml.template @@ -18,10 +18,10 @@ #set $disk_prefix = 'vd' #set $disk_bus = 'virtio' <type>hvm</type> - #end if + #end if #if $getVar('rescue', False) - <kernel>${basepath}/rescue-kernel</kernel> - <initrd>${basepath}/rescue-ramdisk</initrd> + <kernel>${basepath}/kernel.rescue</kernel> + <initrd>${basepath}/ramdisk.rescue</initrd> #else #if $getVar('kernel', None) <kernel>${kernel}</kernel> @@ -47,7 +47,7 @@ #if $getVar('rescue', False) <disk type='file'> <driver type='${driver_type}'/> - <source file='${basepath}/rescue-disk'/> + <source file='${basepath}/disk.rescue'/> <target dev='${disk_prefix}a' bus='${disk_bus}'/> </disk> <disk type='file'> @@ -64,7 +64,7 @@ #if $getVar('local', False) <disk type='file'> <driver type='${driver_type}'/> - <source file='${basepath}/local'/> + <source file='${basepath}/disk.local'/> <target dev='${disk_prefix}b' bus='${disk_bus}'/> </disk> #end if diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index b06246135..e70abb4e5 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -295,7 +295,7 @@ class LibvirtConnection(object): virt_dom.detachDevice(xml) @exception.wrap_exception - def snapshot(self, instance, name): + def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance """ raise NotImplementedError( _("Instance snapshotting is not supported for libvirt" @@ -350,7 +350,7 @@ class LibvirtConnection(object): rescue_images = {'image_id': FLAGS.rescue_image_id, 'kernel_id': FLAGS.rescue_kernel_id, 'ramdisk_id': FLAGS.rescue_ramdisk_id} - self._create_image(instance, xml, 'rescue-', rescue_images) + self._create_image(instance, xml, '.rescue', rescue_images) self._conn.createXML(xml, 0) timer = utils.LoopingCall(f=None) @@ -532,23 +532,23 @@ class LibvirtConnection(object): utils.execute('truncate %s -s %dG' % (target, local_gb)) # TODO(vish): should we format disk by default? - def _create_image(self, inst, libvirt_xml, prefix='', disk_images=None): + def _create_image(self, inst, libvirt_xml, suffix='', disk_images=None): # syntactic nicety - def basepath(fname='', prefix=prefix): + def basepath(fname='', suffix=suffix): return os.path.join(FLAGS.instances_path, inst['name'], - prefix + fname) + fname + suffix) # ensure directories exist and are writable - utils.execute('mkdir -p %s' % basepath(prefix='')) - utils.execute('chmod 0777 %s' % basepath(prefix='')) + utils.execute('mkdir -p %s' % basepath(suffix='')) + utils.execute('chmod 0777 %s' % basepath(suffix='')) LOG.info(_('instance %s: Creating image'), inst['name']) f = open(basepath('libvirt.xml'), 'w') f.write(libvirt_xml) f.close() - # NOTE(vish): No need add the prefix to console.log + # NOTE(vish): No need add the suffix to console.log os.close(os.open(basepath('console.log', ''), os.O_CREAT | os.O_WRONLY, 0660)) @@ -577,7 +577,7 @@ class LibvirtConnection(object): root_fname = disk_images['image_id'] size = FLAGS.minimum_root_size - if inst['instance_type'] == 'm1.tiny' or prefix == 'rescue-': + if inst['instance_type'] == 'm1.tiny' or suffix == '.rescue': size = None root_fname += "_sm" @@ -593,7 +593,7 @@ class LibvirtConnection(object): if type_data['local_gb']: self._cache_image(fn=self._create_local, - target=basepath('local'), + target=basepath('disk.local'), fname="local_%s" % type_data['local_gb'], cow=FLAGS.use_cow_images, local_gb=type_data['local_gb']) @@ -1121,6 +1121,10 @@ class NWFilterFirewall(FirewallDriver): return + def apply_instance_filter(self, instance): + """No-op. Everything is done in prepare_instance_filter""" + pass + def refresh_security_group_rules(self, security_group_id): return self._define_filter( self.security_group_to_nwfilter_xml(security_group_id)) diff --git a/nova/virt/xenapi/fake.py b/nova/virt/xenapi/fake.py index 96d8f5fc8..4bfaf4b57 100644 --- a/nova/virt/xenapi/fake.py +++ b/nova/virt/xenapi/fake.py @@ -76,6 +76,7 @@ def reset(): for c in _CLASSES: _db_content[c] = {} create_host('fake') + create_vm('fake', 'Running', is_a_template=False, is_control_domain=True) def create_host(name_label): @@ -136,14 +137,21 @@ def create_vdi(name_label, read_only, sr_ref, sharable): def create_vbd(vm_ref, vdi_ref): - vbd_rec = {'VM': vm_ref, 'VDI': vdi_ref} + vbd_rec = { + 'VM': vm_ref, + 'VDI': vdi_ref, + 'currently_attached': False, + } vbd_ref = _create_object('VBD', vbd_rec) after_VBD_create(vbd_ref, vbd_rec) return vbd_ref def after_VBD_create(vbd_ref, vbd_rec): - """Create backref from VM to VBD when VBD is created""" + """Create read-only fields and backref from VM to VBD when VBD is + created.""" + vbd_rec['currently_attached'] = False + vbd_rec['device'] = '' vm_ref = vbd_rec['VM'] vm_rec = _db_content['VM'][vm_ref] vm_rec['VBDs'] = [vbd_ref] @@ -152,9 +160,10 @@ def after_VBD_create(vbd_ref, vbd_rec): vbd_rec['vm_name_label'] = vm_name_label -def create_pbd(config, sr_ref, attached): +def create_pbd(config, host_ref, sr_ref, attached): return _create_object('PBD', { 'device-config': config, + 'host': host_ref, 'SR': sr_ref, 'currently-attached': attached, }) @@ -167,6 +176,33 @@ def create_task(name_label): }) +def create_local_srs(): + """Create an SR that looks like the one created on the local disk by + default by the XenServer installer. Do this one per host.""" + for host_ref in _db_content['host'].keys(): + _create_local_sr(host_ref) + + +def _create_local_sr(host_ref): + sr_ref = _create_object('SR', { + 'name_label': 'Local storage', + 'type': 'lvm', + 'content_type': 'user', + 'shared': False, + 'physical_size': str(1 << 30), + 'physical_utilisation': str(0), + 'virtual_allocation': str(0), + 'other_config': { + 'i18n-original-value-name_label': 'Local storage', + 'i18n-key': 'local-storage', + }, + 'VDIs': [] + }) + pbd_ref = create_pbd('', host_ref, sr_ref, True) + _db_content['SR'][sr_ref]['PBDs'] = [pbd_ref] + return sr_ref + + def _create_object(table, obj): ref = str(uuid.uuid4()) obj['uuid'] = str(uuid.uuid4()) @@ -179,9 +215,10 @@ def _create_sr(table, obj): # Forces fake to support iscsi only if sr_type != 'iscsi': raise Failure(['SR_UNKNOWN_DRIVER', sr_type]) + host_ref = _db_content['host'].keys()[0] sr_ref = _create_object(table, obj[2]) vdi_ref = create_vdi('', False, sr_ref, False) - pbd_ref = create_pbd('', sr_ref, True) + pbd_ref = create_pbd('', host_ref, sr_ref, True) _db_content['SR'][sr_ref]['VDIs'] = [vdi_ref] _db_content['SR'][sr_ref]['PBDs'] = [pbd_ref] _db_content['VDI'][vdi_ref]['SR'] = sr_ref @@ -233,6 +270,20 @@ class SessionBase(object): def __init__(self, uri): self._session = None + def VBD_plug(self, _1, ref): + rec = get_record('VBD', ref) + if rec['currently_attached']: + raise Failure(['DEVICE_ALREADY_ATTACHED', ref]) + rec['currently_attached'] = True + rec['device'] = rec['userdevice'] + + def VBD_unplug(self, _1, ref): + rec = get_record('VBD', ref) + if not rec['currently_attached']: + raise Failure(['DEVICE_ALREADY_DETACHED', ref]) + rec['currently_attached'] = False + rec['device'] = '' + def xenapi_request(self, methodname, params): if methodname.startswith('login'): self._login(methodname, params) @@ -289,6 +340,8 @@ class SessionBase(object): return lambda *params: self._getter(name, params) elif self._is_create(name): return lambda *params: self._create(name, params) + elif self._is_destroy(name): + return lambda *params: self._destroy(name, params) else: return None @@ -299,10 +352,16 @@ class SessionBase(object): bits[1].startswith(getter and 'get_' or 'set_')) def _is_create(self, name): + return self._is_method(name, 'create') + + def _is_destroy(self, name): + return self._is_method(name, 'destroy') + + def _is_method(self, name, meth): bits = name.split('.') return (len(bits) == 2 and bits[0] in _CLASSES and - bits[1] == 'create') + bits[1] == meth) def _getter(self, name, params): self._check_session(params) @@ -370,10 +429,9 @@ class SessionBase(object): _create_sr(cls, params) or _create_object(cls, params[1]) # Call hook to provide any fixups needed (ex. creating backrefs) - try: - globals()["after_%s_create" % cls](ref, params[1]) - except KeyError: - pass + after_hook = 'after_%s_create' % cls + if after_hook in globals(): + globals()[after_hook](ref, params[1]) obj = get_record(cls, ref) @@ -383,6 +441,15 @@ class SessionBase(object): return ref + def _destroy(self, name, params): + self._check_session(params) + self._check_arg_count(params, 2) + table, _ = name.split('.') + ref = params[1] + if ref not in _db_content[table]: + raise Failure(['HANDLE_INVALID', table, ref]) + del _db_content[table][ref] + def _async(self, name, params): task_ref = create_task(name) task = _db_content['task'][task_ref] @@ -420,7 +487,7 @@ class SessionBase(object): try: return result[0] except IndexError: - return None + raise Failure(['UUID_INVALID', v, result, recs, k]) return result diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index a91c8ea27..b80ff4dba 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -19,11 +19,14 @@ Helper methods for operations related to the management of VM records and their attributes like VDIs, VIFs, as well as their lookup functions. """ +import os import pickle +import re import urllib from xml.dom import minidom from eventlet import event +import glance.client from nova import exception from nova import flags from nova import log as logging @@ -47,17 +50,23 @@ XENAPI_POWER_STATE = { 'Crashed': power_state.CRASHED} +SECTOR_SIZE = 512 +MBR_SIZE_SECTORS = 63 +MBR_SIZE_BYTES = MBR_SIZE_SECTORS * SECTOR_SIZE +KERNEL_DIR = '/boot/guest' + + class ImageType: - """ - Enumeration class for distinguishing different image types - 0 - kernel/ramdisk image (goes on dom0's filesystem) - 1 - disk image (local SR, partitioned by objectstore plugin) - 2 - raw disk image (local SR, NOT partitioned by plugin) - """ + """ + Enumeration class for distinguishing different image types + 0 - kernel/ramdisk image (goes on dom0's filesystem) + 1 - disk image (local SR, partitioned by objectstore plugin) + 2 - raw disk image (local SR, NOT partitioned by plugin) + """ - KERNEL_RAMDISK = 0 - DISK = 1 - DISK_RAW = 2 + KERNEL_RAMDISK = 0 + DISK = 1 + DISK_RAW = 2 class VMHelper(HelperBase): @@ -207,6 +216,25 @@ class VMHelper(HelperBase): return vif_ref @classmethod + def create_vdi(cls, session, sr_ref, name_label, virtual_size, read_only): + """Create a VDI record and returns its reference.""" + vdi_ref = session.get_xenapi().VDI.create( + {'name_label': name_label, + 'name_description': '', + 'SR': sr_ref, + 'virtual_size': str(virtual_size), + 'type': 'User', + 'sharable': False, + 'read_only': read_only, + 'xenstore_data': {}, + 'other_config': {}, + 'sm_config': {}, + 'tags': []}) + LOG.debug(_('Created VDI %s (%s, %s, %s) on %s.'), vdi_ref, + name_label, virtual_size, read_only, sr_ref) + return vdi_ref + + @classmethod def create_snapshot(cls, session, instance_id, vm_ref, label): """ Creates Snapshot (Template) VM, Snapshot VBD, Snapshot VDI, Snapshot VHD @@ -236,14 +264,15 @@ class VMHelper(HelperBase): return template_vm_ref, [template_vdi_uuid, parent_uuid] @classmethod - def upload_image(cls, session, instance_id, vdi_uuids, image_name): + def upload_image(cls, session, instance_id, vdi_uuids, image_id): """ Requests that the Glance plugin bundle the specified VDIs and push them into Glance using the specified human-friendly name. """ - LOG.debug(_("Asking xapi to upload %s as '%s'"), vdi_uuids, image_name) + logging.debug(_("Asking xapi to upload %s as ID %s"), + vdi_uuids, image_id) params = {'vdi_uuids': vdi_uuids, - 'image_name': image_name, + 'image_id': image_id, 'glance_host': FLAGS.glance_host, 'glance_port': FLAGS.glance_port} @@ -255,15 +284,71 @@ class VMHelper(HelperBase): def fetch_image(cls, session, instance_id, image, user, project, type): """ type is interpreted as an ImageType instance + Related flags: + xenapi_image_service = ['glance', 'objectstore'] + glance_address = 'address for glance services' + glance_port = 'port for glance services' """ - url = images.image_url(image) access = AuthManager().get_access_key(user, project) + + if FLAGS.xenapi_image_service == 'glance': + return cls._fetch_image_glance(session, instance_id, image, + access, type) + else: + return cls._fetch_image_objectstore(session, instance_id, image, + access, user.secret, type) + + @classmethod + def _fetch_image_glance(cls, session, instance_id, image, access, type): + sr = find_sr(session) + if sr is None: + raise exception.NotFound('Cannot find SR to write VDI to') + + c = glance.client.Client(FLAGS.glance_host, FLAGS.glance_port) + + meta, image_file = c.get_image(image) + virtual_size = int(meta['size']) + vdi_size = virtual_size + LOG.debug(_("Size for image %s:%d"), image, virtual_size) + if type == ImageType.DISK: + # Make room for MBR. + vdi_size += MBR_SIZE_BYTES + + vdi = cls.create_vdi(session, sr, _('Glance image %s') % image, + vdi_size, False) + + with_vdi_attached_here(session, vdi, False, + lambda dev: + _stream_disk(dev, type, + virtual_size, image_file)) + if (type == ImageType.KERNEL_RAMDISK): + #we need to invoke a plugin for copying VDI's + #content into proper path + LOG.debug(_("Copying VDI %s to /boot/guest on dom0"), vdi) + fn = "copy_kernel_vdi" + args = {} + args['vdi-ref'] = vdi + #let the plugin copy the correct number of bytes + args['image-size'] = str(vdi_size) + task = session.async_call_plugin('glance', fn, args) + filename = session.wait_for_task(instance_id, task) + #remove the VDI as it is not needed anymore + session.get_xenapi().VDI.destroy(vdi) + LOG.debug(_("Kernel/Ramdisk VDI %s destroyed"), vdi) + return filename + else: + return session.get_xenapi().VDI.get_uuid(vdi) + + @classmethod + def _fetch_image_objectstore(cls, session, instance_id, image, access, + secret, type): + url = images.image_url(image) LOG.debug(_("Asking xapi to fetch %s as %s"), url, access) fn = (type != ImageType.KERNEL_RAMDISK) and 'get_vdi' or 'get_kernel' args = {} args['src_url'] = url args['username'] = access - args['password'] = user.secret + args['password'] = secret args['add_partition'] = 'false' args['raw'] = 'false' if type != ImageType.KERNEL_RAMDISK: @@ -275,14 +360,21 @@ class VMHelper(HelperBase): return uuid @classmethod - def lookup_image(cls, session, vdi_ref): + def lookup_image(cls, session, instance_id, vdi_ref): + if FLAGS.xenapi_image_service == 'glance': + return cls._lookup_image_glance(session, vdi_ref) + else: + return cls._lookup_image_objectstore(session, instance_id, vdi_ref) + + @classmethod + def _lookup_image_objectstore(cls, session, instance_id, vdi_ref): LOG.debug(_("Looking up vdi %s for PV kernel"), vdi_ref) fn = "is_vdi_pv" args = {} args['vdi-ref'] = vdi_ref - #TODO: Call proper function in plugin task = session.async_call_plugin('objectstore', fn, args) - pv_str = session.wait_for_task(task) + pv_str = session.wait_for_task(instance_id, task) + pv = None if pv_str.lower() == 'true': pv = True elif pv_str.lower() == 'false': @@ -291,6 +383,23 @@ class VMHelper(HelperBase): return pv @classmethod + def _lookup_image_glance(cls, session, vdi_ref): + LOG.debug(_("Looking up vdi %s for PV kernel"), vdi_ref) + + def is_vdi_pv(dev): + LOG.debug(_("Running pygrub against %s"), dev) + output = os.popen('pygrub -qn /dev/%s' % dev) + for line in output.readlines(): + #try to find kernel string + m = re.search('(?<=kernel:)/.*(?:>)', line) + if m and m.group(0).find('xen') != -1: + LOG.debug(_("Found Xen kernel %s") % m.group(0)) + return True + LOG.debug(_("No Xen kernel found. Booting HVM.")) + return False + return with_vdi_attached_here(session, vdi_ref, True, is_vdi_pv) + + @classmethod def lookup(cls, session, i): """Look the instance i up, and returns it if available""" vms = session.get_xenapi().VM.get_by_name_label(i) @@ -424,9 +533,16 @@ def wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref, * parent_vhd snapshot """ - #TODO(sirp): we need to timeout this req after a while + max_attempts = FLAGS.xenapi_vhd_coalesce_max_attempts + attempts = {'counter': 0} def _poll_vhds(): + attempts['counter'] += 1 + if attempts['counter'] > max_attempts: + msg = (_("VHD coalesce attempts exceeded (%d > %d), giving up...") + % (attempts['counter'], max_attempts)) + raise exception.Error(msg) + scan_sr(session, instance_id, sr_ref) parent_uuid = get_vhd_parent_uuid(session, vdi_ref) if original_parent_uuid and (parent_uuid != original_parent_uuid): @@ -434,13 +550,12 @@ def wait_for_vhd_coalesce(session, instance_id, sr_ref, vdi_ref, "waiting for coalesce..."), parent_uuid, original_parent_uuid) else: - done.send(parent_uuid) + # Breakout of the loop (normally) and return the parent_uuid + raise utils.LoopingCallDone(parent_uuid) - done = event.Event() loop = utils.LoopingCall(_poll_vhds) loop.start(FLAGS.xenapi_vhd_coalesce_poll_interval, now=True) - parent_uuid = done.wait() - loop.stop() + parent_uuid = loop.wait() return parent_uuid @@ -457,3 +572,123 @@ def get_vdi_for_vm_safely(session, vm_ref): vdi_ref = vdi_refs[0] vdi_rec = session.get_xenapi().VDI.get_record(vdi_ref) return vdi_ref, vdi_rec + + +def find_sr(session): + host = session.get_xenapi_host() + srs = session.get_xenapi().SR.get_all() + for sr in srs: + sr_rec = session.get_xenapi().SR.get_record(sr) + if not ('i18n-key' in sr_rec['other_config'] and + sr_rec['other_config']['i18n-key'] == 'local-storage'): + continue + for pbd in sr_rec['PBDs']: + pbd_rec = session.get_xenapi().PBD.get_record(pbd) + if pbd_rec['host'] == host: + return sr + return None + + +def with_vdi_attached_here(session, vdi, read_only, f): + this_vm_ref = get_this_vm_ref(session) + vbd_rec = {} + vbd_rec['VM'] = this_vm_ref + vbd_rec['VDI'] = vdi + vbd_rec['userdevice'] = 'autodetect' + vbd_rec['bootable'] = False + vbd_rec['mode'] = read_only and 'RO' or 'RW' + vbd_rec['type'] = 'disk' + vbd_rec['unpluggable'] = True + vbd_rec['empty'] = False + vbd_rec['other_config'] = {} + vbd_rec['qos_algorithm_type'] = '' + vbd_rec['qos_algorithm_params'] = {} + vbd_rec['qos_supported_algorithms'] = [] + LOG.debug(_('Creating VBD for VDI %s ... '), vdi) + vbd = session.get_xenapi().VBD.create(vbd_rec) + LOG.debug(_('Creating VBD for VDI %s done.'), vdi) + try: + LOG.debug(_('Plugging VBD %s ... '), vbd) + session.get_xenapi().VBD.plug(vbd) + LOG.debug(_('Plugging VBD %s done.'), vbd) + return f(session.get_xenapi().VBD.get_device(vbd)) + finally: + LOG.debug(_('Destroying VBD for VDI %s ... '), vdi) + vbd_unplug_with_retry(session, vbd) + ignore_failure(session.get_xenapi().VBD.destroy, vbd) + LOG.debug(_('Destroying VBD for VDI %s done.'), vdi) + + +def vbd_unplug_with_retry(session, vbd): + """Call VBD.unplug on the given VBD, with a retry if we get + DEVICE_DETACH_REJECTED. For reasons which I don't understand, we're + seeing the device still in use, even when all processes using the device + should be dead.""" + while True: + try: + session.get_xenapi().VBD.unplug(vbd) + LOG.debug(_('VBD.unplug successful first time.')) + return + except VMHelper.XenAPI.Failure, e: + if (len(e.details) > 0 and + e.details[0] == 'DEVICE_DETACH_REJECTED'): + LOG.debug(_('VBD.unplug rejected: retrying...')) + time.sleep(1) + elif (len(e.details) > 0 and + e.details[0] == 'DEVICE_ALREADY_DETACHED'): + LOG.debug(_('VBD.unplug successful eventually.')) + return + else: + LOG.error(_('Ignoring XenAPI.Failure in VBD.unplug: %s'), + e) + return + + +def ignore_failure(func, *args, **kwargs): + try: + return func(*args, **kwargs) + except VMHelper.XenAPI.Failure, e: + LOG.error(_('Ignoring XenAPI.Failure %s'), e) + return None + + +def get_this_vm_uuid(): + with file('/sys/hypervisor/uuid') as f: + return f.readline().strip() + + +def get_this_vm_ref(session): + return session.get_xenapi().VM.get_by_uuid(get_this_vm_uuid()) + + +def _stream_disk(dev, type, virtual_size, image_file): + offset = 0 + if type == ImageType.DISK: + offset = MBR_SIZE_BYTES + _write_partition(virtual_size, dev) + + with open('/dev/%s' % dev, 'wb') as f: + f.seek(offset) + for chunk in image_file: + f.write(chunk) + + +def _write_partition(virtual_size, dev): + dest = '/dev/%s' % dev + mbr_last = MBR_SIZE_SECTORS - 1 + primary_first = MBR_SIZE_SECTORS + primary_last = MBR_SIZE_SECTORS + (virtual_size / SECTOR_SIZE) - 1 + + LOG.debug(_('Writing partition table %d %d to %s...'), + primary_first, primary_last, dest) + + def execute(cmd, process_input=None, check_exit_code=True): + return utils.execute(cmd=cmd, + process_input=process_input, + check_exit_code=check_exit_code) + + execute('parted --script %s mklabel msdos' % dest) + execute('parted --script %s mkpart primary %ds %ds' % + (dest, primary_first, primary_last)) + + LOG.debug(_('Writing partition table %s done.'), dest) diff --git a/nova/virt/xenapi/vmops.py b/nova/virt/xenapi/vmops.py index 50ee24325..729e20d63 100644 --- a/nova/virt/xenapi/vmops.py +++ b/nova/virt/xenapi/vmops.py @@ -85,7 +85,8 @@ class VMOps(object): #Have a look at the VDI and see if it has a PV kernel pv_kernel = False if not instance.kernel_id: - pv_kernel = VMHelper.lookup_image(self._session, vdi_ref) + pv_kernel = VMHelper.lookup_image(self._session, instance.id, + vdi_ref) kernel = None if instance.kernel_id: kernel = VMHelper.fetch_image(self._session, instance.id, @@ -161,11 +162,11 @@ class VMOps(object): raise Exception(_('Instance not present %s') % instance_name) return vm - def snapshot(self, instance, name): + def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance :param instance: instance to be snapshotted - :param name: name/label to be given to the snapshot + :param image_id: id of image to upload to Steps involved in a XenServer snapshot: @@ -201,7 +202,7 @@ class VMOps(object): try: # call plugin to ship snapshot off to glance VMHelper.upload_image( - self._session, instance.id, template_vdi_uuids, name) + self._session, instance.id, template_vdi_uuids, image_id) finally: self._destroy(instance, template_vm_ref, shutdown=False) diff --git a/nova/virt/xenapi_conn.py b/nova/virt/xenapi_conn.py index c850dbf80..438571338 100644 --- a/nova/virt/xenapi_conn.py +++ b/nova/virt/xenapi_conn.py @@ -89,10 +89,17 @@ flags.DEFINE_float('xenapi_task_poll_interval', 'The interval used for polling of remote tasks ' '(Async.VM.start, etc). Used only if ' 'connection_type=xenapi.') +flags.DEFINE_string('xenapi_image_service', + 'glance', + 'Where to get VM images: glance or objectstore.') flags.DEFINE_float('xenapi_vhd_coalesce_poll_interval', 5.0, 'The interval used for polling of coalescing vhds.' ' Used only if connection_type=xenapi.') +flags.DEFINE_integer('xenapi_vhd_coalesce_max_attempts', + 5, + 'Max number of times to poll for VHD to coalesce.' + ' Used only if connection_type=xenapi.') flags.DEFINE_string('target_host', None, 'iSCSI Target Host') @@ -141,9 +148,9 @@ class XenAPIConnection(object): """Create VM instance""" self._vmops.spawn(instance) - def snapshot(self, instance, name): + def snapshot(self, instance, image_id): """ Create snapshot from a running VM instance """ - self._vmops.snapshot(instance, name) + self._vmops.snapshot(instance, image_id) def resize(self, instance, flavor): """Resize a VM instance""" diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 71fe18a40..5fefa10cf 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -47,7 +47,7 @@ flags.DEFINE_integer('iscsi_num_targets', 'Number of iscsi target ids per host') flags.DEFINE_string('iscsi_target_prefix', 'iqn.2010-10.org.openstack:', 'prefix for iscsi volumes') -flags.DEFINE_string('iscsi_ip_prefix', '127.0', +flags.DEFINE_string('iscsi_ip_prefix', '$my_ip', 'discover volumes on the ip that starts with this prefix') flags.DEFINE_string('rbd_pool', 'rbd', 'the rbd pool in which volumes are stored') @@ -285,7 +285,8 @@ class ISCSIDriver(VolumeDriver): self._execute("sudo iscsiadm -m node -T %s -p %s --op update " "-n node.startup -v automatic" % (iscsi_name, iscsi_portal)) - return "/dev/iscsi/%s" % volume['name'] + return "/dev/disk/by-path/ip-%s-iscsi-%s-lun-0" % (iscsi_portal, + iscsi_name) def undiscover_volume(self, volume): """Undiscover volume on a remote host.""" diff --git a/nova/wsgi.py b/nova/wsgi.py index 00b351253..4f5307d80 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -21,6 +21,7 @@ Utility methods for working with WSGI servers """ +import os import sys from xml.dom import minidom @@ -33,10 +34,16 @@ import webob import webob.dec import webob.exc +from paste import deploy + +from nova import flags from nova import log as logging from nova import utils +FLAGS = flags.FLAGS + + class WritableLogger(object): """A thin wrapper that responds to `write` and logs.""" @@ -76,10 +83,33 @@ class Server(object): class Application(object): -# TODO(gundlach): I think we should toss this class, now that it has no -# purpose. """Base WSGI application wrapper. Subclasses need to implement __call__.""" + @classmethod + def factory(cls, global_config, **local_config): + """Used for paste app factories in paste.deploy config fles. + + Any local configuration (that is, values under the [app:APPNAME] + section of the paste config) will be passed into the `__init__` method + as kwargs. + + A hypothetical configuration would look like: + + [app:wadl] + latest_version = 1.3 + paste.app_factory = nova.api.fancy_api:Wadl.factory + + which would result in a call to the `Wadl` class as + + import nova.api.fancy_api + fancy_api.Wadl(latest_version='1.3') + + You could of course re-implement the `factory` method in subclasses, + but using the kwarg passing it shouldn't be necessary. + + """ + return cls(**local_config) + def __call__(self, environ, start_response): r"""Subclasses will probably want to implement __call__ like this: @@ -125,6 +155,33 @@ class Middleware(Application): behavior. """ + @classmethod + def factory(cls, global_config, **local_config): + """Used for paste app factories in paste.deploy config fles. + + Any local configuration (that is, values under the [filter:APPNAME] + section of the paste config) will be passed into the `__init__` method + as kwargs. + + A hypothetical configuration would look like: + + [filter:analytics] + redis_host = 127.0.0.1 + paste.filter_factory = nova.api.analytics:Analytics.factory + + which would result in a call to the `Analytics` class as + + import nova.api.analytics + analytics.Analytics(app_from_paste, redis_host='127.0.0.1') + + You could of course re-implement the `factory` method in subclasses, + but using the kwarg passing it shouldn't be necessary. + + """ + def _factory(app): + return cls(app, **local_config) + return _factory + def __init__(self, application): self.application = application @@ -403,3 +460,64 @@ class Serializer(object): node = doc.createTextNode(str(data)) result.appendChild(node) return result + + +def paste_config_file(basename): + """Find the best location in the system for a paste config file. + + Search Order + ------------ + + The search for a paste config file honors `FLAGS.state_path`, which in a + version checked out from bzr will be the `nova` directory in the top level + of the checkout, and in an installation for a package for your distribution + will likely point to someplace like /etc/nova. + + This method tries to load places likely to be used in development or + experimentation before falling back to the system-wide configuration + in `/etc/nova/`. + + * Current working directory + * the `etc` directory under state_path, because when working on a checkout + from bzr this will point to the default + * top level of FLAGS.state_path, for distributions + * /etc/nova, which may not be diffrerent from state_path on your distro + + """ + + configfiles = [basename, + os.path.join(FLAGS.state_path, 'etc', basename), + os.path.join(FLAGS.state_path, basename), + '/etc/nova/%s' % basename] + for configfile in configfiles: + if os.path.exists(configfile): + return configfile + + +def load_paste_configuration(filename, appname): + """Returns a paste configuration dict, or None.""" + filename = os.path.abspath(filename) + config = None + try: + config = deploy.appconfig("config:%s" % filename, name=appname) + except LookupError: + pass + return config + + +def load_paste_app(filename, appname): + """Builds a wsgi app from a paste config, None if app not configured.""" + filename = os.path.abspath(filename) + app = None + try: + app = deploy.loadapp("config:%s" % filename, name=appname) + except LookupError: + pass + return app + + +def paste_config_to_flags(config, mixins): + for k, v in mixins.iteritems(): + value = config.get(k, v) + converted_value = FLAGS[k].parser.Parse(value) + setattr(FLAGS, k, converted_value) diff --git a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance index 5e648b970..aadacce57 100644 --- a/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance +++ b/plugins/xenserver/xenapi/etc/xapi.d/plugins/glance @@ -18,7 +18,7 @@ # under the License. # -# XenAPI plugin for putting images into glance +# XenAPI plugin for managing glance images # import base64 @@ -40,29 +40,57 @@ from pluginlib_nova import * configure_logging('glance') CHUNK_SIZE = 8192 +KERNEL_DIR = '/boot/guest' FILE_SR_PATH = '/var/run/sr-mount' +def copy_kernel_vdi(session,args): + vdi = exists(args, 'vdi-ref') + size = exists(args,'image-size') + #Use the uuid as a filename + vdi_uuid=session.xenapi.VDI.get_uuid(vdi) + copy_args={'vdi_uuid':vdi_uuid,'vdi_size':int(size)} + filename=with_vdi_in_dom0(session, vdi, False, + lambda dev: + _copy_kernel_vdi('/dev/%s' % dev,copy_args)) + return filename + +def _copy_kernel_vdi(dest,copy_args): + vdi_uuid=copy_args['vdi_uuid'] + vdi_size=copy_args['vdi_size'] + logging.debug("copying kernel/ramdisk file from %s to /boot/guest/%s",dest,vdi_uuid) + filename=KERNEL_DIR + '/' + vdi_uuid + #read data from /dev/ and write into a file on /boot/guest + of=open(filename,'wb') + f=open(dest,'rb') + #copy only vdi_size bytes + data=f.read(vdi_size) + of.write(data) + f.close() + of.close() + logging.debug("Done. Filename: %s",filename) + return filename + def put_vdis(session, args): params = pickle.loads(exists(args, 'params')) vdi_uuids = params["vdi_uuids"] - image_name = params["image_name"] + image_id = params["image_id"] glance_host = params["glance_host"] glance_port = params["glance_port"] sr_path = get_sr_path(session) #FIXME(sirp): writing to a temp file until Glance supports chunked-PUTs - tmp_file = "%s.tar.gz" % os.path.join('/tmp', image_name) + tmp_file = "%s.tar.gz" % os.path.join('/tmp', str(image_id)) tar_cmd = ['tar', '-zcf', tmp_file, '--directory=%s' % sr_path] paths = [ "%s.vhd" % vdi_uuid for vdi_uuid in vdi_uuids ] tar_cmd.extend(paths) logging.debug("Bundling image with cmd: %s", tar_cmd) subprocess.call(tar_cmd) logging.debug("Writing to test file %s", tmp_file) - put_bundle_in_glance(tmp_file, image_name, glance_host, glance_port) + put_bundle_in_glance(tmp_file, image_id, glance_host, glance_port) return "" # FIXME(sirp): return anything useful here? -def put_bundle_in_glance(tmp_file, image_name, glance_host, glance_port): +def put_bundle_in_glance(tmp_file, image_id, glance_host, glance_port): size = os.path.getsize(tmp_file) basename = os.path.basename(tmp_file) @@ -72,7 +100,6 @@ def put_bundle_in_glance(tmp_file, image_name, glance_host, glance_port): 'x-image-meta-store': 'file', 'x-image-meta-is_public': 'True', 'x-image-meta-type': 'raw', - 'x-image-meta-name': image_name, 'x-image-meta-size': size, 'content-length': size, 'content-type': 'application/octet-stream', @@ -80,7 +107,7 @@ def put_bundle_in_glance(tmp_file, image_name, glance_host, glance_port): conn = httplib.HTTPConnection(glance_host, glance_port) #NOTE(sirp): httplib under python2.4 won't accept a file-like object # to request - conn.putrequest('POST', '/images') + conn.putrequest('PUT', '/images/%s' % image_id) for header, value in headers.iteritems(): conn.putheader(header, value) @@ -129,4 +156,5 @@ def find_sr(session): if __name__ == '__main__': - XenAPIPlugin.dispatch({'put_vdis': put_vdis}) + XenAPIPlugin.dispatch({'put_vdis': put_vdis, + 'copy_kernel_vdi': copy_kernel_vdi}) diff --git a/run_tests.py b/run_tests.py index 5b8617f63..7b5e2192a 100644 --- a/run_tests.py +++ b/run_tests.py @@ -17,7 +17,7 @@ # See the License for the specific language governing permissions and # limitations under the License. - +import gettext import os import unittest import sys diff --git a/tools/pip-requires b/tools/pip-requires index 3aa76c24d..895e81eb3 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -25,4 +25,6 @@ bzr Twisted>=10.1.0 PasteDeploy paste +sqlalchemy-migrate netaddr +glance |
