From c524508bc58aa561b81c3133526c981cce835a59 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 20 Sep 2010 01:50:08 -0700 Subject: added rescue mode support and made reboot work from any state --- nova/compute/manager.py | 37 +++++++++--- nova/virt/fake.py | 12 ++++ nova/virt/libvirt.rescue.qemu.xml.template | 33 +++++++++++ nova/virt/libvirt.rescue.uml.xml.template | 26 +++++++++ nova/virt/libvirt_conn.py | 94 +++++++++++++++++++++++++----- 5 files changed, 179 insertions(+), 23 deletions(-) create mode 100644 nova/virt/libvirt.rescue.qemu.xml.template create mode 100644 nova/virt/libvirt.rescue.uml.xml.template diff --git a/nova/compute/manager.py b/nova/compute/manager.py index 954227b42..56e08f881 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -122,17 +122,8 @@ class ComputeManager(manager.Manager): @exception.wrap_exception def reboot_instance(self, context, instance_id): """Reboot an instance on this server.""" - self._update_state(context, instance_id) instance_ref = self.db.instance_get(context, instance_id) - if instance_ref['state'] != power_state.RUNNING: - raise exception.Error( - 'trying to reboot a non-running' - 'instance: %s (state: %s excepted: %s)' % - (instance_ref['str_id'], - instance_ref['state'], - power_state.RUNNING)) - logging.debug('instance %s: rebooting', instance_ref['name']) self.db.instance_set_state(context, instance_id, @@ -141,6 +132,34 @@ class ComputeManager(manager.Manager): yield self.driver.reboot(instance_ref) self._update_state(context, instance_id) + @defer.inlineCallbacks + @exception.wrap_exception + def rescue_instance(self, context, instance_id): + """Rescue an instance on this server.""" + instance_ref = self.db.instance_get(context, instance_id) + + logging.debug('instance %s: rescuing', instance_ref['name']) + self.db.instance_set_state(context, + instance_id, + power_state.NOSTATE, + 'rescuing') + yield self.driver.rescue(instance_ref) + self._update_state(context, instance_id) + + @defer.inlineCallbacks + @exception.wrap_exception + def unrescue_instance(self, context, instance_id): + """Rescue an instance on this server.""" + instance_ref = self.db.instance_get(context, instance_id) + + logging.debug('instance %s: unrescuing', instance_ref['name']) + self.db.instance_set_state(context, + instance_id, + power_state.NOSTATE, + 'unrescuing') + yield self.driver.unrescue(instance_ref) + self._update_state(context, instance_id) + @exception.wrap_exception def get_console_output(self, context, instance_id): """Send the console output for an instance.""" diff --git a/nova/virt/fake.py b/nova/virt/fake.py index 4ae6afcc4..3e88060f6 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -119,6 +119,18 @@ class FakeConnection(object): """ return defer.succeed(None) + def rescue(self, instance): + """ + Rescue the specified instance. + """ + return defer.succeed(None) + + def unrescue(self, instance): + """ + Unrescue the specified instance. + """ + return defer.succeed(None) + def destroy(self, instance): """ Destroy (shutdown and delete) the specified instance. diff --git a/nova/virt/libvirt.rescue.qemu.xml.template b/nova/virt/libvirt.rescue.qemu.xml.template new file mode 100644 index 000000000..164326452 --- /dev/null +++ b/nova/virt/libvirt.rescue.qemu.xml.template @@ -0,0 +1,33 @@ + + %(name)s + + hvm + %(basepath)s/rescue-kernel + %(basepath)s/rescue-ramdisk + root=/dev/vda1 console=ttyS0 + + + + + %(memory_kb)s + %(vcpus)s + + + + + + + + + + + + + + + + + + + + diff --git a/nova/virt/libvirt.rescue.uml.xml.template b/nova/virt/libvirt.rescue.uml.xml.template new file mode 100644 index 000000000..836f47532 --- /dev/null +++ b/nova/virt/libvirt.rescue.uml.xml.template @@ -0,0 +1,26 @@ + + %(name)s + %(memory_kb)s + + %(type)s + /usr/bin/linux + /dev/ubda1 + + + + + + + + + + + + + + + + + + + diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index d868e083c..b9edc8e85 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -44,6 +44,16 @@ libxml2 = None FLAGS = flags.FLAGS +flags.DEFINE_string('libvirt_rescue_xml_template', + utils.abspath('virt/libvirt.rescue.qemu.xml.template'), + 'Libvirt RESCUE XML Template for QEmu/KVM') +flags.DEFINE_string('libvirt_rescue_uml_xml_template', + utils.abspath('virt/libvirt.rescue.uml.xml.template'), + 'Libvirt RESCUE XML Template for user-mode-linux') +# TODO(vish): These flags should probably go into a shared location +flags.DEFINE_string('rescue_image_id', 'ami-rescue', 'Rescue ami image') +flags.DEFINE_string('rescue_kernel_id', 'aki-rescue', 'Rescue aki image') +flags.DEFINE_string('rescue_ramdisk_id', 'ari-rescue', 'Rescue ari image') flags.DEFINE_string('libvirt_xml_template', utils.abspath('virt/libvirt.qemu.xml.template'), 'Libvirt XML Template for QEmu/KVM') @@ -76,9 +86,12 @@ def get_connection(read_only): class LibvirtConnection(object): def __init__(self, read_only): - self.libvirt_uri, template_file = self.get_uri_and_template() + (self.libvirt_uri, + template_file, + rescue_file)= self.get_uri_and_templates() self.libvirt_xml = open(template_file).read() + self.rescue_xml = open(rescue_file).read() self._wrapped_conn = None self.read_only = read_only @@ -100,14 +113,16 @@ class LibvirtConnection(object): return False raise - def get_uri_and_template(self): + def get_uri_and_templates(self): if FLAGS.libvirt_type == 'uml': uri = FLAGS.libvirt_uri or 'uml:///system' template_file = FLAGS.libvirt_uml_xml_template + rescue_file = FLAGS.libvirt_rescue_uml_xml_template else: uri = FLAGS.libvirt_uri or 'qemu:///system' template_file = FLAGS.libvirt_xml_template - return uri, template_file + rescue_file = FLAGS.libvirt_rescue_xml_template + return uri, template_file, rescue_file def _connect(self, uri, read_only): auth = [[libvirt.VIR_CRED_AUTHNAME, libvirt.VIR_CRED_NOECHOPROMPT], @@ -123,7 +138,7 @@ class LibvirtConnection(object): return [self._conn.lookupByID(x).name() for x in self._conn.listDomainsID()] - def destroy(self, instance): + def destroy(self, instance, cleanup=True): try: virt_dom = self._conn.lookupByName(instance['name']) virt_dom.destroy() @@ -131,7 +146,8 @@ class LibvirtConnection(object): pass # If the instance is already terminated, we're still happy d = defer.Deferred() - d.addCallback(lambda _: self._cleanup(instance)) + if cleanup: + d.addCallback(lambda _: self._cleanup(instance)) # FIXME: What does this comment mean? # TODO(termie): short-circuit me for tests # WE'LL save this for when we do shutdown, @@ -181,8 +197,8 @@ class LibvirtConnection(object): @defer.inlineCallbacks @exception.wrap_exception def reboot(self, instance): + yield self.destroy(instance, False) xml = self.to_xml(instance) - yield self._conn.lookupByName(instance['name']).destroy() yield self._conn.createXML(xml, 0) d = defer.Deferred() @@ -206,6 +222,46 @@ class LibvirtConnection(object): timer.start(interval=0.5, now=True) yield d + @defer.inlineCallbacks + @exception.wrap_exception + def rescue(self, instance): + yield self.destroy(instance, False) + + xml = self.to_xml(instance, rescue=True) + rescue_images = {'image_id': FLAGS.rescue_image_id, + 'kernel_id': FLAGS.rescue_kernel_id, + 'ramdisk_id': FLAGS.rescue_ramdisk_id} + yield self._create_image(instance, xml, 'rescue-', rescue_images) + yield self._conn.createXML(xml, 0) + + d = defer.Deferred() + timer = task.LoopingCall(f=None) + def _wait_for_rescue(): + try: + state = self.get_info(instance['name'])['state'] + db.instance_set_state(None, instance['id'], state) + if state == power_state.RUNNING: + logging.debug('instance %s: rescued', instance['name']) + timer.stop() + d.callback(None) + except Exception, exn: + logging.error('_wait_for_rescue failed: %s', exn) + db.instance_set_state(None, + instance['id'], + power_state.SHUTDOWN) + timer.stop() + d.callback(None) + timer.f = _wait_for_rescue + timer.start(interval=0.5, now=True) + yield d + + @defer.inlineCallbacks + @exception.wrap_exception + def unrescue(self, instance): + # NOTE(vish): Because reboot destroys and recreates an instance using + # the normal xml file, we can just call reboot here + yield self.reboot(instance) + @defer.inlineCallbacks @exception.wrap_exception def spawn(self, instance): @@ -243,15 +299,16 @@ class LibvirtConnection(object): yield local_d @defer.inlineCallbacks - def _create_image(self, inst, libvirt_xml): + def _create_image(self, inst, libvirt_xml, prefix='', disk_images=None): # syntactic nicety - basepath = lambda fname='': os.path.join(FLAGS.instances_path, + basepath = lambda fname='', prefix=prefix: os.path.join( + FLAGS.instances_path, inst['name'], - fname) + prefix + fname) # ensure directories exist and are writable - yield process.simple_execute('mkdir -p %s' % basepath()) - yield process.simple_execute('chmod 0777 %s' % basepath()) + yield process.simple_execute('mkdir -p %s' % basepath(prefix='')) + yield process.simple_execute('chmod 0777 %s' % basepath(prefix='')) # TODO(termie): these are blocking calls, it would be great @@ -261,11 +318,17 @@ class LibvirtConnection(object): f.write(libvirt_xml) f.close() - os.close(os.open(basepath('console.log'), os.O_CREAT | os.O_WRONLY, 0660)) + # NOTE(vish): No need add the prefix to console.log + os.close(os.open(basepath('console.log', ''), + os.O_CREAT | os.O_WRONLY, 0660)) user = manager.AuthManager().get_user(inst['user_id']) project = manager.AuthManager().get_project(inst['project_id']) + if not disk_images: + disk_images = {'image_id': inst['image_id'], + 'kernel_id': inst['kernel_id'], + 'ramdisk_id': inst['ramdisk_id']} if not os.path.exists(basepath('disk')): yield images.fetch(inst.image_id, basepath('disk-raw'), user, project) if not os.path.exists(basepath('kernel')): @@ -311,7 +374,7 @@ class LibvirtConnection(object): yield process.simple_execute('sudo chown root %s' % basepath('disk')) - def to_xml(self, instance): + def to_xml(self, instance, rescue=False): # TODO(termie): cache? logging.debug('instance %s: starting toXML method', instance['name']) network = db.project_get_network(None, instance['project_id']) @@ -325,7 +388,10 @@ class LibvirtConnection(object): 'vcpus': instance_type['vcpus'], 'bridge_name': network['bridge'], 'mac_address': instance['mac_address']} - libvirt_xml = self.libvirt_xml % xml_info + if rescue: + libvirt_xml = self.rescue_xml % xml_info + else: + libvirt_xml = self.libvirt_xml % xml_info logging.debug('instance %s: finished toXML method', instance['name']) return libvirt_xml -- cgit From 86a8a5a2091dc0a36ccbe05f46609c82b1060291 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 19 Oct 2010 19:37:42 -0400 Subject: Fixes https://bugs.launchpad.net/nova/+bug/663551 by catching exceptions at the top level of the API and turning them into Faults. --- nova/api/openstack/__init__.py | 11 ++++++ nova/tests/api/openstack/test_api.py | 65 ++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 nova/tests/api/openstack/test_api.py diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 5706dbc09..24e1dd090 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -23,6 +23,7 @@ WSGI middleware for OpenStack API controllers. import json import time +import logging import routes import webob.dec import webob.exc @@ -53,6 +54,16 @@ class API(wsgi.Middleware): app = AuthMiddleware(RateLimitingMiddleware(APIRouter())) super(API, self).__init__(app) + @webob.dec.wsgify + def __call__(self, req): + try: + return req.get_response(self.application) + except Exception as ex: + logging.warn("Caught error: %s" % str(ex)) + exc = webob.exc.HTTPInternalServerError(explanation=str(ex)) + return faults.Fault(exc) + + class AuthMiddleware(wsgi.Middleware): """Authorize the openstack API request or return an HTTP Forbidden.""" diff --git a/nova/tests/api/openstack/test_api.py b/nova/tests/api/openstack/test_api.py new file mode 100644 index 000000000..171818806 --- /dev/null +++ b/nova/tests/api/openstack/test_api.py @@ -0,0 +1,65 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest +import webob.exc +import webob.dec + +import nova.api.openstack +from nova.api.openstack import API +from nova.api.openstack import faults +from webob import Request + +class APITest(unittest.TestCase): + + def test_exceptions_are_converted_to_faults(self): + @webob.dec.wsgify + def succeed(req): + return 'Succeeded' + @webob.dec.wsgify + def raise_webob_exc(req): + raise webob.exc.HTTPNotFound(explanation='Raised a webob.exc') + @webob.dec.wsgify + def fail(req): + raise Exception("Threw an exception") + @webob.dec.wsgify + def raise_api_fault(req): + exc = webob.exc.HTTPNotFound(explanation='Raised a webob.exc') + return faults.Fault(exc) + api = API() + + api.application = succeed + resp = Request.blank('/').get_response(api) + self.assertFalse('cloudServersFault' in resp.body, resp.body) + self.assertEqual(resp.status_int, 200, resp.body) + + api.application = raise_webob_exc + resp = Request.blank('/').get_response(api) + self.assertFalse('cloudServersFault' in resp.body, resp.body) + self.assertEqual(resp.status_int, 404, resp.body) + + api.application = raise_api_fault + resp = Request.blank('/').get_response(api) + self.assertTrue('itemNotFound' in resp.body, resp.body) + self.assertEqual(resp.status_int, 404, resp.body) + + api.application = fail + resp = Request.blank('/').get_response(api) + self.assertTrue('cloudServersFault' in resp.body, resp.body) + self.assertEqual(resp.status_int, 500, resp.body) + + -- cgit From d9deffcba4ff6ab3b5a6459c9d626dc4f5334336 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 19 Oct 2010 19:39:20 -0400 Subject: Add unit test for XML requests converting errors to Faults --- nova/tests/api/openstack/test_api.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/nova/tests/api/openstack/test_api.py b/nova/tests/api/openstack/test_api.py index 171818806..a8c0ff9f8 100644 --- a/nova/tests/api/openstack/test_api.py +++ b/nova/tests/api/openstack/test_api.py @@ -59,7 +59,10 @@ class APITest(unittest.TestCase): api.application = fail resp = Request.blank('/').get_response(api) - self.assertTrue('cloudServersFault' in resp.body, resp.body) + self.assertTrue('{"cloudServersFault' in resp.body, resp.body) self.assertEqual(resp.status_int, 500, resp.body) - + api.application = fail + resp = Request.blank('/.xml').get_response(api) + self.assertTrue(' Date: Tue, 19 Oct 2010 19:40:57 -0400 Subject: Authorize Image before download. --- nova/objectstore/handler.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index b93e92fe6..5945ebbce 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -318,6 +318,8 @@ class ImageResource(ErrorHandlingResource): def render_GET(self, request): """Returns the image file""" + if not self.img.is_authorized(request.context, True): + raise exception.NotAuthorized return static.File(self.img.image_path, defaultType='application/octet-stream' ).render_GET(request) -- cgit From a75eabea4800a1c10487bc2962aeb26aaea26f8d Mon Sep 17 00:00:00 2001 From: Todd Willey Date: Tue, 19 Oct 2010 19:57:24 -0400 Subject: Construct exception instead of raising a class. --- nova/objectstore/handler.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index 5945ebbce..cc0d8ffc9 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -119,7 +119,7 @@ def get_context(request): # Authorization Header format: 'AWS :' authorization_header = request.getHeader('Authorization') if not authorization_header: - raise exception.NotAuthorized + raise exception.NotAuthorized() auth_header_value = authorization_header.split(' ')[1] access, _ignored, secret = auth_header_value.rpartition(':') am = manager.AuthManager() @@ -134,7 +134,7 @@ def get_context(request): return context.RequestContext(user, project) except exception.Error as ex: logging.debug("Authentication Failure: %s", ex) - raise exception.NotAuthorized + raise exception.NotAuthorized() class ErrorHandlingResource(resource.Resource): """Maps exceptions to 404 / 401 codes. Won't work for @@ -209,7 +209,7 @@ class BucketResource(ErrorHandlingResource): return error.NoResource(message="No such bucket").render(request) if not bucket_object.is_authorized(request.context): - raise exception.NotAuthorized + raise exception.NotAuthorized() prefix = get_argument(request, "prefix", u"") marker = get_argument(request, "marker", u"") @@ -239,7 +239,7 @@ class BucketResource(ErrorHandlingResource): bucket_object = bucket.Bucket(self.name) if not bucket_object.is_authorized(request.context): - raise exception.NotAuthorized + raise exception.NotAuthorized() bucket_object.delete() request.setResponseCode(204) @@ -262,7 +262,7 @@ class ObjectResource(ErrorHandlingResource): logging.debug("Getting object: %s / %s", self.bucket.name, self.name) if not self.bucket.is_authorized(request.context): - raise exception.NotAuthorized + raise exception.NotAuthorized() obj = self.bucket[urllib.unquote(self.name)] request.setHeader("Content-Type", "application/unknown") @@ -280,7 +280,7 @@ class ObjectResource(ErrorHandlingResource): logging.debug("Putting object: %s / %s", self.bucket.name, self.name) if not self.bucket.is_authorized(request.context): - raise exception.NotAuthorized + raise exception.NotAuthorized() key = urllib.unquote(self.name) request.content.seek(0, 0) @@ -301,7 +301,7 @@ class ObjectResource(ErrorHandlingResource): self.name) if not self.bucket.is_authorized(request.context): - raise exception.NotAuthorized + raise exception.NotAuthorized() del self.bucket[urllib.unquote(self.name)] request.setResponseCode(204) @@ -319,7 +319,7 @@ class ImageResource(ErrorHandlingResource): def render_GET(self, request): """Returns the image file""" if not self.img.is_authorized(request.context, True): - raise exception.NotAuthorized + raise exception.NotAuthorized() return static.File(self.img.image_path, defaultType='application/octet-stream' ).render_GET(request) @@ -371,12 +371,12 @@ class ImagesResource(resource.Resource): image_path = os.path.join(FLAGS.images_path, image_id) if not image_path.startswith(FLAGS.images_path) or \ os.path.exists(image_path): - raise exception.NotAuthorized + raise exception.NotAuthorized() bucket_object = bucket.Bucket(image_location.split("/")[0]) if not bucket_object.is_authorized(request.context): - raise exception.NotAuthorized + raise exception.NotAuthorized() p = multiprocessing.Process(target=image.Image.register_aws_image, args=(image_id, image_location, request.context)) @@ -391,7 +391,7 @@ class ImagesResource(resource.Resource): image_object = image.Image(image_id) if not image_object.is_authorized(request.context): logging.debug("not authorized for render_POST in images") - raise exception.NotAuthorized + raise exception.NotAuthorized() operation = get_argument(request, 'operation', u'') if operation: @@ -413,7 +413,7 @@ class ImagesResource(resource.Resource): image_object = image.Image(image_id) if not image_object.is_authorized(request.context): - raise exception.NotAuthorized + raise exception.NotAuthorized() image_object.delete() -- cgit From 5c2bc4a709d3b2f1785ac7fabbb2a1183d20d0d6 Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Wed, 20 Oct 2010 09:50:09 -0500 Subject: Updated documentation --- doc/source/getting.started.rst | 49 +++++++++++------------------------------- doc/source/index.rst | 17 +++++++++------ 2 files changed, 22 insertions(+), 44 deletions(-) diff --git a/doc/source/getting.started.rst b/doc/source/getting.started.rst index f683bb256..4610a9a5a 100644 --- a/doc/source/getting.started.rst +++ b/doc/source/getting.started.rst @@ -18,23 +18,20 @@ Getting Started with Nova ========================= +This code base is continually changing so dependencies also change. -GOTTA HAVE A nova.pth file added or it WONT WORK (will write setup.py file soon) - -Create a file named nova.pth in your python libraries directory -(usually /usr/local/lib/python2.6/dist-packages) with a single line that points -to the directory where you checked out the source (that contains the nova/ -directory). - -DEPENDENCIES +Dependencies ------------ Related servers we rely on * RabbitMQ: messaging queue, used for all communication between components -* OpenLDAP: users, groups (maybe cut) +* OpenLDAP: users, groups + +Optional servers + * ReDIS: Remote Dictionary Store (for fast, shared state data) -* nginx: HTTP server to handle serving large files (because Tornado can't) +* nginx: HTTP server to handle serving large files Python libraries we don't vendor @@ -44,7 +41,6 @@ Python libraries we don't vendor Vendored python libaries (don't require any installation) -* Tornado: scalable non blocking web server for api requests * Twisted: just for the twisted.internet.defer package * boto: python api for aws api * IPy: library for managing ip addresses @@ -60,35 +56,14 @@ Installation -------------- :: - # system libraries and tools - apt-get install -y aoetools vlan curl - modprobe aoe - - # python libraries - apt-get install -y python-setuptools python-dev python-pycurl python-m2crypto - - # ON THE CLOUD CONTROLLER - apt-get install -y rabbitmq-server dnsmasq nginx - # build redis from 2.0.0-rc1 source - # setup ldap (slap.sh as root will remove ldap and reinstall it) - NOVA_PATH/nova/auth/slap.sh - /etc/init.d/rabbitmq-server start - - # ON VOLUME NODE: - apt-get install -y vblade-persist - - # ON THE COMPUTE NODE: - apt-get install -y python-libvirt - apt-get install -y kpartx kvm libvirt-bin - modprobe kvm - - # optional packages - apt-get install -y euca2ools + Due to many changes it's best to rely on the 'OpenStack wiki ' for installation instructions. Configuration --------------- -ON CLOUD CONTROLLER +These instructions are incomplete, but we are actively updating the 'OpenStack wiki ' with more configuration information. + +On the cloud controller * Add yourself to the libvirtd group, log out, and log back in * fix hardcoded ec2 metadata/userdata uri ($IP is the IP of the cloud), and masqurade all traffic from launched instances @@ -137,7 +112,7 @@ Running Launch servers * rabbitmq -* redis +* redis (optional) * slapd * nginx diff --git a/doc/source/index.rst b/doc/source/index.rst index 6627fe066..56f2e6b88 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -15,18 +15,21 @@ License for the specific language governing permissions and limitations under the License. -Welcome to nova's documentation! +Welcome to Nova's documentation! ================================ -Nova is a cloud computing fabric controller (the main part of an IaaS system) built to match the popular AWS EC2 and S3 APIs. -It is written in Python, using the Tornado and Twisted frameworks, and relies on the standard AMQP messaging protocol, -and the Redis distributed KVS. -Nova is intended to be easy to extend, and adapt. For example, it currently uses +Nova is a cloud computing fabric controller (the main part of an IaaS system). +It is written in Python and relies on the standard AMQP messaging protocol, uses the Twisted framework, +and optionally uses the Redis distributed key value store for authorization. + +Nova is intended to be easy to extend and adapt. For example, it currently uses an LDAP server for users and groups, but also includes a fake LDAP server, that stores data in Redis. It has extensive test coverage, and uses the -Sphinx toolkit (the same as Python itself) for code and user documentation. +Sphinx toolkit (the same as Python itself) for code and developer documentation. +Additional documentation is available on the +'OpenStack wiki '_. While Nova is currently in Beta use within several organizations, the codebase -is very much under active development - there are bugs! +is very much under active development - please test it and log bugs! Contents: -- cgit From 08b4a4fb49744c9d3c499a52922a4e2cb2110e14 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 20 Oct 2010 13:46:37 -0700 Subject: make sure looping calls are created after service starts and add some tests to verify service delegation works --- nova/service.py | 53 ++++++++++++++++++++++-------------------- nova/tests/service_unittest.py | 44 ++++++++++++++++++++++++++++++++--- 2 files changed, 69 insertions(+), 28 deletions(-) diff --git a/nova/service.py b/nova/service.py index 2d7961ab9..d53d92b65 100644 --- a/nova/service.py +++ b/nova/service.py @@ -49,15 +49,17 @@ flags.DEFINE_integer('periodic_interval', 60, class Service(object, service.Service): """Base class for workers that run on hosts.""" - def __init__(self, host, binary, topic, manager, *args, **kwargs): + def __init__(self, host, binary, topic, manager, report_interval=None, + periodic_interval=None, *args, **kwargs): self.host = host self.binary = binary self.topic = topic self.manager_class_name = manager + self.report_interval = report_interval + self.periodic_interval = periodic_interval super(Service, self).__init__(*args, **kwargs) self.saved_args, self.saved_kwargs = args, kwargs - def startService(self): # pylint: disable-msg C0103 manager_class = utils.import_class(self.manager_class_name) self.manager = manager_class(host=self.host, *self.saved_args, @@ -73,6 +75,26 @@ class Service(object, service.Service): except exception.NotFound: self._create_service_ref(ctxt) + conn = rpc.Connection.instance() + if self.report_interval: + consumer_all = rpc.AdapterConsumer( + connection=conn, + topic=self.topic, + proxy=self) + consumer_node = rpc.AdapterConsumer( + connection=conn, + topic='%s.%s' % (self.topic, self.host), + proxy=self) + + consumer_all.attach_to_twisted() + consumer_node.attach_to_twisted() + + pulse = task.LoopingCall(self.report_state) + pulse.start(interval=self.report_interval, now=False) + + if self.periodic_interval: + pulse = task.LoopingCall(self.periodic_tasks) + pulse.start(interval=self.periodic_interval, now=False) def _create_service_ref(self, context): service_ref = db.service_create(context, @@ -83,10 +105,8 @@ class Service(object, service.Service): self.service_id = service_ref['id'] def __getattr__(self, key): - try: - return super(Service, self).__getattr__(key) - except AttributeError: - return getattr(self.manager, key) + manager = self.__dict__.get('manager', None) + return getattr(manager, key) @classmethod def create(cls, @@ -119,25 +139,8 @@ class Service(object, service.Service): if not periodic_interval: periodic_interval = FLAGS.periodic_interval logging.warn("Starting %s node", topic) - service_obj = cls(host, binary, topic, manager) - conn = rpc.Connection.instance() - consumer_all = rpc.AdapterConsumer( - connection=conn, - topic=topic, - proxy=service_obj) - consumer_node = rpc.AdapterConsumer( - connection=conn, - topic='%s.%s' % (topic, host), - proxy=service_obj) - - consumer_all.attach_to_twisted() - consumer_node.attach_to_twisted() - - pulse = task.LoopingCall(service_obj.report_state) - pulse.start(interval=report_interval, now=False) - - pulse = task.LoopingCall(service_obj.periodic_tasks) - pulse.start(interval=periodic_interval, now=False) + service_obj = cls(host, binary, topic, manager, + report_interval, periodic_interval) # This is the parent service that twistd will be looking for when it # parses this file, return it so that we can get it into globals. diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index 61db52742..8533fab0a 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -39,11 +39,44 @@ flags.DEFINE_string("fake_manager", "nova.tests.service_unittest.FakeManager", class FakeManager(manager.Manager): """Fake manager for tests""" - pass + def test_method(self): + return 'manager' + + +class ExtendedService(service.Service): + def test_method(self): + return 'service' + + +class ServiceManagerTestCase(test.BaseTestCase): + """Test cases for Services""" + + def test_attribute_error_for_no_manager(self): + serv = service.Service('test', + 'test', + 'test', + 'nova.tests.service_unittest.FakeManager') + self.assertRaises(AttributeError, getattr, serv, 'test_method') + + def test_message_gets_to_manager(self): + serv = service.Service('test', + 'test', + 'test', + 'nova.tests.service_unittest.FakeManager') + serv.startService() + self.assertEqual(serv.test_method(), 'manager') + + def test_override_manager_method(self): + serv = ExtendedService('test', + 'test', + 'test', + 'nova.tests.service_unittest.FakeManager') + serv.startService() + self.assertEqual(serv.test_method(), 'service') class ServiceTestCase(test.BaseTestCase): - """Test cases for rpc""" + """Test cases for Services""" def setUp(self): # pylint: disable=C0103 super(ServiceTestCase, self).setUp() @@ -54,6 +87,11 @@ class ServiceTestCase(test.BaseTestCase): host = 'foo' binary = 'nova-fake' topic = 'fake' + + # NOTE(vish): Create was moved out of mox replay to make sure that + # the looping calls are created in StartService. + app = service.Service.create(host=host, binary=binary) + self.mox.StubOutWithMock(rpc, 'AdapterConsumer', use_mock_anything=True) @@ -99,7 +137,6 @@ class ServiceTestCase(test.BaseTestCase): service_create).AndReturn(service_ref) self.mox.ReplayAll() - app = service.Service.create(host=host, binary=binary) startApplication(app, False) self.assert_(app) @@ -190,3 +227,4 @@ class ServiceTestCase(test.BaseTestCase): rv = yield s.report_state(host, binary) self.assert_(not s.model_disconnected) + -- cgit From d454bff397c29651e20fdea105b3e8cc197d0f5e Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Wed, 20 Oct 2010 13:54:53 -0700 Subject: Check the pid to make sure it refers to the correct dnsmasq process --- nova/network/linux_net.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index c0be0e8cc..f75a079d9 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -167,11 +167,9 @@ def get_dhcp_hosts(context, network_id): return '\n'.join(hosts) -# TODO(ja): if the system has restarted or pid numbers have wrapped -# then you cannot be certain that the pid refers to the -# dnsmasq. As well, sending a HUP only reloads the hostfile, -# so any configuration options (like dchp-range, vlan, ...) -# aren't reloaded +# NOTE(ja): Sending a HUP only reloads the hostfile, so any +# configuration options (like dchp-range, vlan, ...) +# aren't reloaded. def update_dhcp(context, network_id): """(Re)starts a dnsmasq server for a given network @@ -191,13 +189,15 @@ def update_dhcp(context, network_id): # if dnsmasq is already running, then tell it to reload if pid: - # TODO(ja): use "/proc/%d/cmdline" % (pid) to determine if pid refers - # correct dnsmasq process - try: - _execute('sudo kill -HUP %d' % pid) - return - except Exception as exc: # pylint: disable-msg=W0703 - logging.debug("Hupping dnsmasq threw %s", exc) + out, _err = _execute('cat /proc/%d/cmdline' % pid, check_exit_code=False) + if conffile in out: + try: + _execute('sudo kill -HUP %d' % pid) + return + except Exception as exc: # pylint: disable-msg=W0703 + logging.debug("Hupping dnsmasq threw %s", exc) + else: + logging.debug("Pid %d is stale, relaunching dnsmasq", pid) # FLAGFILE and DNSMASQ_INTERFACE in env env = {'FLAGFILE': FLAGS.dhcpbridge_flagfile, -- cgit From 7233ed2e35e3829b05a6b229af9d3a243da744fb Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Wed, 20 Oct 2010 17:06:51 -0500 Subject: Made updates based on review comments. --- doc/source/getting.started.rst | 22 ++++++++++------------ doc/source/index.rst | 11 ++++++----- 2 files changed, 16 insertions(+), 17 deletions(-) diff --git a/doc/source/getting.started.rst b/doc/source/getting.started.rst index 4610a9a5a..2df4a45ea 100644 --- a/doc/source/getting.started.rst +++ b/doc/source/getting.started.rst @@ -26,22 +26,22 @@ Dependencies Related servers we rely on * RabbitMQ: messaging queue, used for all communication between components -* OpenLDAP: users, groups Optional servers -* ReDIS: Remote Dictionary Store (for fast, shared state data) -* nginx: HTTP server to handle serving large files +* OpenLDAP: By default, the auth server uses the RDBMS-backed datastore by setting FLAGS.auth_driver to 'nova.auth.dbdriver.DbDriver'. But OpenLDAP (or LDAP) could be configured. +* ReDIS: By default, this is not enabled as the auth driver. Python libraries we don't vendor * M2Crypto: python library interface for openssl * curl -* XenAPI: Needed only for Xen Cloud Platform or XenServer support. Available from http://wiki.xensource.com/xenwiki/XCP_SDK or http://community.citrix.com/cdn/xs/sdks. +* XenAPI: Needed only for Xen Cloud Platform or XenServer support. Available from http://wiki.xensource.com/xenwiki/XCP_SDK or http://community.citrix.com/cdn/xs/sdks. Vendored python libaries (don't require any installation) * Twisted: just for the twisted.internet.defer package +* Tornado: scalable non blocking web server for api requests * boto: python api for aws api * IPy: library for managing ip addresses @@ -54,19 +54,19 @@ Recommended Installation -------------- -:: - Due to many changes it's best to rely on the 'OpenStack wiki ' for installation instructions. + Due to many changes it's best to rely on the `OpenStack wiki `_ for installation instructions. Configuration --------------- -These instructions are incomplete, but we are actively updating the 'OpenStack wiki ' with more configuration information. +These instructions are incomplete, but we are actively updating the `OpenStack wiki `_ with more configuration information. On the cloud controller * Add yourself to the libvirtd group, log out, and log back in -* fix hardcoded ec2 metadata/userdata uri ($IP is the IP of the cloud), and masqurade all traffic from launched instances +* Fix hardcoded ec2 metadata/userdata uri ($IP is the IP of the cloud), and masqurade all traffic from launched instances + :: iptables -t nat -A PREROUTING -s 0.0.0.0/0 -d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT --to-destination $IP:8773 @@ -94,9 +94,9 @@ On the cloud controller } } -ON VOLUME NODE +On the volume node -* create a filesystem (you can use an actual disk if you have one spare, default is /dev/sdb) +* Create a filesystem (you can use an actual disk if you have one spare, default is /dev/sdb) :: @@ -113,8 +113,6 @@ Launch servers * rabbitmq * redis (optional) -* slapd -* nginx Launch nova components diff --git a/doc/source/index.rst b/doc/source/index.rst index 56f2e6b88..1109e9011 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -22,11 +22,12 @@ Nova is a cloud computing fabric controller (the main part of an IaaS system). It is written in Python and relies on the standard AMQP messaging protocol, uses the Twisted framework, and optionally uses the Redis distributed key value store for authorization. -Nova is intended to be easy to extend and adapt. For example, it currently uses -an LDAP server for users and groups, but also includes a fake LDAP server, -that stores data in Redis. It has extensive test coverage, and uses the -Sphinx toolkit (the same as Python itself) for code and developer documentation. -Additional documentation is available on the +Nova is intended to be easy to extend and adapt. For example, authentication and authorization +requests by default use an RDBMS-backed datastore driver. However, there is already support +for using LDAP backing authentication (slapd) and if you wish to "fake" LDAP, there is a module +available that uses ReDIS to store authentication information in an LDAP-like backing datastore. +It has extensive test coverage, and uses the Sphinx toolkit (the same as Python itself) for code +and developer documentation. Additional documentation is available on the 'OpenStack wiki '_. While Nova is currently in Beta use within several organizations, the codebase is very much under active development - please test it and log bugs! -- cgit From 943e8bcda4f304caa15d689d4db0e50376860f00 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Thu, 21 Oct 2010 08:42:15 -0700 Subject: validate device in AttachDisk --- nova/api/ec2/cloud.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 6d4f58499..bd7fbc3ac 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -25,6 +25,7 @@ datastore. import base64 import datetime import logging +import re import os import time @@ -533,6 +534,8 @@ class CloudController(object): def attach_volume(self, context, volume_id, instance_id, device, **kwargs): volume_ref = db.volume_get_by_ec2_id(context, volume_id) + if not re.match("^/dev/[a-z]d[a-z]+$", device): + raise exception.ApiError("Invalid device. Example /dev/vdb") # TODO(vish): abstract status checking? if volume_ref['status'] != "available": raise exception.ApiError("Volume status must be available") -- cgit From e012a2b73725bded6d4ac39747d57affda545770 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 21 Oct 2010 11:49:51 -0700 Subject: PEP8 and pylint cleanup. There should be no functional changes here, just style changes to get violations down. --- bin/nova-api | 1 + bin/nova-manage | 47 +++++++++--------- bzrplugins/novalog/__init__.py | 19 ++++---- nova/adminclient.py | 97 ++++++++++++++------------------------ nova/context.py | 11 +++-- nova/crypto.py | 45 +++++++++++------- nova/exception.py | 5 +- nova/fakerabbit.py | 1 - nova/flags.py | 3 +- nova/manager.py | 8 ++-- nova/process.py | 23 +++++---- nova/quota.py | 3 +- nova/rpc.py | 2 + nova/server.py | 7 ++- nova/test.py | 23 +++++---- nova/tests/api/__init__.py | 4 +- nova/tests/api_unittest.py | 2 +- nova/tests/compute_unittest.py | 4 +- nova/tests/network_unittest.py | 4 +- nova/tests/objectstore_unittest.py | 8 ++-- nova/tests/quota_unittest.py | 4 +- nova/tests/rpc_unittest.py | 2 +- nova/tests/scheduler_unittest.py | 6 +-- nova/tests/service_unittest.py | 2 +- nova/tests/volume_unittest.py | 2 +- nova/twistd.py | 19 +++++--- nova/utils.py | 23 +++++---- nova/validate.py | 41 +++++++++------- nova/wsgi.py | 22 +++++---- pylintrc | 2 +- 30 files changed, 233 insertions(+), 207 deletions(-) diff --git a/bin/nova-api b/bin/nova-api index a5027700b..20f1bd74f 100755 --- a/bin/nova-api +++ b/bin/nova-api @@ -39,6 +39,7 @@ from nova import server FLAGS = flags.FLAGS flags.DEFINE_integer('api_port', 8773, 'API port') + def main(_args): from nova import api from nova import wsgi diff --git a/bin/nova-manage b/bin/nova-manage index 219f32250..08b3da123 100755 --- a/bin/nova-manage +++ b/bin/nova-manage @@ -22,8 +22,8 @@ # Copyright (c) 2005, the Lawrence Journal-World # All rights reserved. # -# Redistribution and use in source and binary forms, with or without modification, -# are permitted provided that the following conditions are met: +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: # # 1. Redistributions of source code must retain the above copyright notice, # this list of conditions and the following disclaimer. @@ -32,20 +32,21 @@ # notice, this list of conditions and the following disclaimer in the # documentation and/or other materials provided with the distribution. # -# 3. Neither the name of Django nor the names of its contributors may be used -# to endorse or promote products derived from this software without +# 3. Neither the name of Django nor the names of its contributors may be +# used to endorse or promote products derived from this software without # specific prior written permission. # -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. """ @@ -181,8 +182,8 @@ class ShellCommands(object): if shell == 'ipython': try: import IPython - # Explicitly pass an empty list as arguments, because otherwise IPython - # would use sys.argv from this script. + # Explicitly pass an empty list as arguments, because + # otherwise IPython would use sys.argv from this script. shell = IPython.Shell.IPShell(argv=[]) shell.mainloop() except ImportError: @@ -190,13 +191,14 @@ class ShellCommands(object): if shell == 'python': import code - try: # Try activating rlcompleter, because it's handy. + try: + # Try activating rlcompleter, because it's handy. import readline except ImportError: pass else: - # We don't have to wrap the following import in a 'try', because - # we already know 'readline' was imported successfully. + # We don't have to wrap the following import in a 'try', + # because we already know 'readline' was imported successfully. import rlcompleter readline.parse_and_bind("tab:complete") code.interact() @@ -242,7 +244,6 @@ class UserCommands(object): print 'export EC2_ACCESS_KEY=%s' % user.access print 'export EC2_SECRET_KEY=%s' % user.secret - def __init__(self): self.manager = manager.AuthManager() @@ -291,6 +292,7 @@ class UserCommands(object): is_admin = False self.manager.modify_user(name, access_key, secret_key, is_admin) + class ProjectCommands(object): """Class for managing projects.""" @@ -380,7 +382,6 @@ class FloatingIpCommands(object): db.floating_ip_destroy(context.get_admin_context(), str(address)) - def list(self, host=None): """Lists all floating ips (optionally by host) arguments: [host]""" @@ -397,6 +398,7 @@ class FloatingIpCommands(object): floating_ip['address'], instance) + class NetworkCommands(object): """Class for managing networks.""" @@ -429,8 +431,7 @@ CATEGORIES = [ ('shell', ShellCommands), ('vpn', VpnCommands), ('floating', FloatingIpCommands), - ('network', NetworkCommands) -] + ('network', NetworkCommands)] def lazy_match(name, key_value_tuples): diff --git a/bzrplugins/novalog/__init__.py b/bzrplugins/novalog/__init__.py index e16b2e00f..9817dc886 100644 --- a/bzrplugins/novalog/__init__.py +++ b/bzrplugins/novalog/__init__.py @@ -17,13 +17,14 @@ import bzrlib.log from bzrlib.osutils import format_date -# -# This is mostly stolen from bzrlib.log.GnuChangelogLogFormatter -# The difference is that it logs the author rather than the committer -# which for Nova always is Tarmac. -# + class NovaLogFormat(bzrlib.log.GnuChangelogLogFormatter): + """This is mostly stolen from bzrlib.log.GnuChangelogLogFormatter + The difference is that it logs the author rather than the committer + which for Nova always is Tarmac.""" + preferred_levels = 1 + def log_revision(self, revision): """Log a revision, either merged or not.""" to_file = self.to_file @@ -38,13 +39,14 @@ class NovaLogFormat(bzrlib.log.GnuChangelogLogFormatter): to_file.write('%s %s\n\n' % (date_str, ", ".join(authors))) if revision.delta is not None and revision.delta.has_changed(): - for c in revision.delta.added + revision.delta.removed + revision.delta.modified: + for c in revision.delta.added + revision.delta.removed + \ + revision.delta.modified: path, = c[:1] to_file.write('\t* %s:\n' % (path,)) for c in revision.delta.renamed: - oldpath,newpath = c[:2] + oldpath, newpath = c[:2] # For renamed files, show both the old and the new path - to_file.write('\t* %s:\n\t* %s:\n' % (oldpath,newpath)) + to_file.write('\t* %s:\n\t* %s:\n' % (oldpath, newpath)) to_file.write('\n') if not revision.rev.message: @@ -56,4 +58,3 @@ class NovaLogFormat(bzrlib.log.GnuChangelogLogFormatter): to_file.write('\n') bzrlib.log.register_formatter('novalog', NovaLogFormat) - diff --git a/nova/adminclient.py b/nova/adminclient.py index fc9fcfde0..b7a3d2c32 100644 --- a/nova/adminclient.py +++ b/nova/adminclient.py @@ -25,10 +25,10 @@ import httplib from boto.ec2.regioninfo import RegionInfo -DEFAULT_CLC_URL='http://127.0.0.1:8773' -DEFAULT_REGION='nova' -DEFAULT_ACCESS_KEY='admin' -DEFAULT_SECRET_KEY='admin' +DEFAULT_CLC_URL = 'http://127.0.0.1:8773' +DEFAULT_REGION = 'nova' +DEFAULT_ACCESS_KEY = 'admin' +DEFAULT_SECRET_KEY = 'admin' class UserInfo(object): @@ -199,9 +199,7 @@ class NovaAdminClient(object): def connection_for(self, username, project, clc_url=None, region=None, **kwargs): - """ - Returns a boto ec2 connection for the given username. - """ + """Returns a boto ec2 connection for the given username.""" if not clc_url: clc_url = self.clc_url if not region: @@ -220,36 +218,37 @@ class NovaAdminClient(object): **kwargs) def split_clc_url(self, clc_url): - """ - Splits a cloud controller endpoint url. - """ + """Splits a cloud controller endpoint url.""" parts = httplib.urlsplit(clc_url) is_secure = parts.scheme == 'https' ip, port = parts.netloc.split(':') return {'ip': ip, 'port': int(port), 'is_secure': is_secure} def get_users(self): - """ grabs the list of all users """ + """Grabs the list of all users.""" return self.apiconn.get_list('DescribeUsers', {}, [('item', UserInfo)]) def get_user(self, name): - """ grab a single user by name """ - user = self.apiconn.get_object('DescribeUser', {'Name': name}, UserInfo) - + """Grab a single user by name.""" + user = self.apiconn.get_object('DescribeUser', {'Name': name}, + UserInfo) if user.username != None: return user def has_user(self, username): - """ determine if user exists """ + """Determine if user exists.""" return self.get_user(username) != None def create_user(self, username): - """ creates a new user, returning the userinfo object with access/secret """ - return self.apiconn.get_object('RegisterUser', {'Name': username}, UserInfo) + """Creates a new user, returning the userinfo object with + access/secret.""" + return self.apiconn.get_object('RegisterUser', {'Name': username}, + UserInfo) def delete_user(self, username): - """ deletes a user """ - return self.apiconn.get_object('DeregisterUser', {'Name': username}, UserInfo) + """Deletes a user.""" + return self.apiconn.get_object('DeregisterUser', {'Name': username}, + UserInfo) def get_roles(self, project_roles=True): """Returns a list of available roles.""" @@ -258,11 +257,10 @@ class NovaAdminClient(object): [('item', UserRole)]) def get_user_roles(self, user, project=None): - """Returns a list of roles for the given user. - Omitting project will return any global roles that the user has. - Specifying project will return only project specific roles. - """ - params = {'User':user} + """Returns a list of roles for the given user. Omitting project will + return any global roles that the user has. Specifying project will + return only project specific roles.""" + params = {'User': user} if project: params['Project'] = project return self.apiconn.get_list('DescribeUserRoles', @@ -270,24 +268,19 @@ class NovaAdminClient(object): [('item', UserRole)]) def add_user_role(self, user, role, project=None): - """ - Add a role to a user either globally or for a specific project. - """ + """Add a role to a user either globally or for a specific project.""" return self.modify_user_role(user, role, project=project, operation='add') def remove_user_role(self, user, role, project=None): - """ - Remove a role from a user either globally or for a specific project. - """ + """Remove a role from a user either globally or for a specific + project.""" return self.modify_user_role(user, role, project=project, operation='remove') def modify_user_role(self, user, role, project=None, operation='add', **kwargs): - """ - Add or remove a role for a user and project. - """ + """Add or remove a role for a user and project.""" params = {'User': user, 'Role': role, 'Project': project, @@ -295,9 +288,7 @@ class NovaAdminClient(object): return self.apiconn.get_status('ModifyUserRole', params) def get_projects(self, user=None): - """ - Returns a list of all projects. - """ + """Returns a list of all projects.""" if user: params = {'User': user} else: @@ -307,9 +298,7 @@ class NovaAdminClient(object): [('item', ProjectInfo)]) def get_project(self, name): - """ - Returns a single project with the specified name. - """ + """Returns a single project with the specified name.""" project = self.apiconn.get_object('DescribeProject', {'Name': name}, ProjectInfo) @@ -319,9 +308,7 @@ class NovaAdminClient(object): def create_project(self, projectname, manager_user, description=None, member_users=None): - """ - Creates a new project. - """ + """Creates a new project.""" params = {'Name': projectname, 'ManagerUser': manager_user, 'Description': description, @@ -329,50 +316,38 @@ class NovaAdminClient(object): return self.apiconn.get_object('RegisterProject', params, ProjectInfo) def delete_project(self, projectname): - """ - Permanently deletes the specified project. - """ + """Permanently deletes the specified project.""" return self.apiconn.get_object('DeregisterProject', {'Name': projectname}, ProjectInfo) def get_project_members(self, name): - """ - Returns a list of members of a project. - """ + """Returns a list of members of a project.""" return self.apiconn.get_list('DescribeProjectMembers', {'Name': name}, [('item', ProjectMember)]) def add_project_member(self, user, project): - """ - Adds a user to a project. - """ + """Adds a user to a project.""" return self.modify_project_member(user, project, operation='add') def remove_project_member(self, user, project): - """ - Removes a user from a project. - """ + """Removes a user from a project.""" return self.modify_project_member(user, project, operation='remove') def modify_project_member(self, user, project, operation='add'): - """ - Adds or removes a user from a project. - """ + """Adds or removes a user from a project.""" params = {'User': user, 'Project': project, 'Operation': operation} return self.apiconn.get_status('ModifyProjectMember', params) def get_zip(self, user, project): - """ - Returns the content of a zip file containing novarc and access credentials. - """ + """Returns the content of a zip file containing novarc and access + credentials.""" params = {'Name': user, 'Project': project} zip = self.apiconn.get_object('GenerateX509ForUser', params, UserInfo) return zip.file def get_hosts(self): return self.apiconn.get_list('DescribeHosts', {}, [('item', HostInfo)]) - diff --git a/nova/context.py b/nova/context.py index f5d3fed08..f2669c9f1 100644 --- a/nova/context.py +++ b/nova/context.py @@ -26,7 +26,9 @@ import random from nova import exception from nova import utils + class RequestContext(object): + def __init__(self, user, project, is_admin=None, read_deleted=False, remote_address=None, timestamp=None, request_id=None): if hasattr(user, 'id'): @@ -56,10 +58,8 @@ class RequestContext(object): timestamp = utils.parse_isotime(timestamp) self.timestamp = timestamp if not request_id: - request_id = ''.join( - [random.choice('ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-') - for x in xrange(20)] - ) + chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890-' + request_id = ''.join([random.choice(chars) for x in xrange(20)]) self.request_id = request_id @property @@ -81,7 +81,8 @@ class RequestContext(object): from nova.auth import manager if not self._project: try: - self._project = manager.AuthManager().get_project(self.project_id) + auth_manager = manager.AuthManager() + self._project = auth_manager.get_project(self.project_id) except exception.NotFound: pass return self._project diff --git a/nova/crypto.py b/nova/crypto.py index 1c6fe57ad..16b4f5e1f 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -39,9 +39,12 @@ from nova import flags FLAGS = flags.FLAGS flags.DEFINE_string('ca_file', 'cacert.pem', 'Filename of root CA') -flags.DEFINE_string('keys_path', utils.abspath('../keys'), 'Where we keep our keys') -flags.DEFINE_string('ca_path', utils.abspath('../CA'), 'Where we keep our root CA') -flags.DEFINE_boolean('use_intermediate_ca', False, 'Should we use intermediate CAs for each project?') +flags.DEFINE_string('keys_path', utils.abspath('../keys'), + 'Where we keep our keys') +flags.DEFINE_string('ca_path', utils.abspath('../CA'), + 'Where we keep our root CA') +flags.DEFINE_boolean('use_intermediate_ca', False, + 'Should we use intermediate CAs for each project?') def ca_path(project_id): @@ -55,11 +58,11 @@ def fetch_ca(project_id=None, chain=True): project_id = None buffer = "" if project_id: - with open(ca_path(project_id),"r") as cafile: + with open(ca_path(project_id), "r") as cafile: buffer += cafile.read() if not chain: return buffer - with open(ca_path(None),"r") as cafile: + with open(ca_path(None), "r") as cafile: buffer += cafile.read() return buffer @@ -88,17 +91,18 @@ def generate_key_pair(bits=1024): def ssl_pub_to_ssh_pub(ssl_public_key, name='root', suffix='nova'): - rsa_key = M2Crypto.RSA.load_pub_key_bio(M2Crypto.BIO.MemoryBuffer(ssl_public_key)) + pub_key_buffer = M2Crypto.BIO.MemoryBuffer(ssl_public_key) + rsa_key = M2Crypto.RSA.load_pub_key_bio(pub_key_buffer) e, n = rsa_key.pub() key_type = 'ssh-rsa' key_data = struct.pack('>I', len(key_type)) key_data += key_type - key_data += '%s%s' % (e,n) + key_data += '%s%s' % (e, n) b64_blob = base64.b64encode(key_data) - return '%s %s %s@%s\n' %(key_type, b64_blob, name, suffix) + return '%s %s %s@%s\n' % (key_type, b64_blob, name, suffix) def generate_x509_cert(subject, bits=1024): @@ -106,8 +110,11 @@ def generate_x509_cert(subject, bits=1024): keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key')) csrfile = os.path.join(tmpdir, 'temp.csr') logging.debug("openssl genrsa -out %s %s" % (keyfile, bits)) - utils.runthis("Generating private key: %s", "openssl genrsa -out %s %s" % (keyfile, bits)) - utils.runthis("Generating CSR: %s", "openssl req -new -key %s -out %s -batch -subj %s" % (keyfile, csrfile, subject)) + utils.runthis("Generating private key: %s", + "openssl genrsa -out %s %s" % (keyfile, bits)) + utils.runthis("Generating CSR: %s", + "openssl req -new -key %s -out %s -batch -subj %s" % + (keyfile, csrfile, subject)) private_key = open(keyfile).read() csr = open(csrfile).read() shutil.rmtree(tmpdir) @@ -123,7 +130,8 @@ def sign_csr(csr_text, intermediate=None): if not os.path.exists(user_ca): start = os.getcwd() os.chdir(FLAGS.ca_path) - utils.runthis("Generating intermediate CA: %s", "sh geninter.sh %s" % (intermediate)) + utils.runthis("Generating intermediate CA: %s", + "sh geninter.sh %s" % (intermediate)) os.chdir(start) return _sign_csr(csr_text, user_ca) @@ -137,7 +145,10 @@ def _sign_csr(csr_text, ca_folder): start = os.getcwd() # Change working dir to CA os.chdir(ca_folder) - utils.runthis("Signing cert: %s", "openssl ca -batch -out %s/outbound.crt -config ./openssl.cnf -infiles %s/inbound.csr" % (tmpfolder, tmpfolder)) + utils.runthis("Signing cert: %s", + "openssl ca -batch -out %s/outbound.crt " + "-config ./openssl.cnf -infiles %s/inbound.csr" % + (tmpfolder, tmpfolder)) os.chdir(start) with open("%s/outbound.crt" % (tmpfolder), "r") as crtfile: return crtfile.read() @@ -148,10 +159,11 @@ def mkreq(bits, subject="foo", ca=0): req = M2Crypto.X509.Request() rsa = M2Crypto.RSA.gen_key(bits, 65537, callback=lambda: None) pk.assign_rsa(rsa) - rsa = None # should not be freed here + # Should not be freed here + rsa = None req.set_pubkey(pk) req.set_subject(subject) - req.sign(pk,'sha512') + req.sign(pk, 'sha512') assert req.verify(pk) pk2 = req.get_pubkey() assert req.verify(pk2) @@ -165,7 +177,8 @@ def mkcacert(subject='nova', years=1): cert = M2Crypto.X509.X509() cert.set_serial_number(1) cert.set_version(2) - cert.set_subject(sub) # FIXME subject is not set in mkreq yet + # FIXME subject is not set in mkreq yet + cert.set_subject(sub) t = long(time.time()) + time.timezone now = M2Crypto.ASN1.ASN1_UTCTIME() now.set_time(t) @@ -189,7 +202,6 @@ def mkcacert(subject='nova', years=1): return cert, pk, pkey - # Copyright (c) 2006-2009 Mitch Garnaat http://garnaat.org/ # # Permission is hereby granted, free of charge, to any person obtaining a @@ -212,6 +224,7 @@ def mkcacert(subject='nova', years=1): # IN THE SOFTWARE. # http://code.google.com/p/boto + def compute_md5(fp): """ @type fp: file diff --git a/nova/exception.py b/nova/exception.py index f157fab2d..6d6c37338 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -47,7 +47,7 @@ class ApiError(Error): def __init__(self, message='Unknown', code='Unknown'): self.message = message self.code = code - super(ApiError, self).__init__('%s: %s'% (code, message)) + super(ApiError, self).__init__('%s: %s' % (code, message)) class NotFound(Error): @@ -69,6 +69,7 @@ class NotEmpty(Error): class Invalid(Error): pass + class InvalidInputException(Error): pass @@ -86,5 +87,3 @@ def wrap_exception(f): raise _wrap.func_name = f.func_name return _wrap - - diff --git a/nova/fakerabbit.py b/nova/fakerabbit.py index df5e61e6e..c64617931 100644 --- a/nova/fakerabbit.py +++ b/nova/fakerabbit.py @@ -130,7 +130,6 @@ class Backend(object): self._exchanges[exchange].publish( message, routing_key=routing_key) - __instance = None def __init__(self, *args, **kwargs): diff --git a/nova/flags.py b/nova/flags.py index 3b473488f..f3b0384ad 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -180,7 +180,8 @@ DEFINE_string('connection_type', 'libvirt', 'libvirt, xenapi or fake') DEFINE_integer('s3_port', 3333, 's3 port') DEFINE_string('s3_host', '127.0.0.1', 's3 host') DEFINE_string('compute_topic', 'compute', 'the topic compute nodes listen on') -DEFINE_string('scheduler_topic', 'scheduler', 'the topic scheduler nodes listen on') +DEFINE_string('scheduler_topic', 'scheduler', + 'the topic scheduler nodes listen on') DEFINE_string('volume_topic', 'volume', 'the topic volume nodes listen on') DEFINE_string('network_topic', 'network', 'the topic network nodes listen on') diff --git a/nova/manager.py b/nova/manager.py index 56ba7d3f6..4244b2db4 100644 --- a/nova/manager.py +++ b/nova/manager.py @@ -45,8 +45,6 @@ class Manager(object): yield def init_host(self): - """Do any initialization that needs to be run if this is a standalone service. - - Child classes should override this method. - """ - pass + """Do any initialization that needs to be run if this is a standalone + service. Child classes should override this method.""" + pass diff --git a/nova/process.py b/nova/process.py index 13cb90e82..b33df048b 100644 --- a/nova/process.py +++ b/nova/process.py @@ -36,6 +36,7 @@ FLAGS = flags.FLAGS flags.DEFINE_integer('process_pool_size', 4, 'Number of processes to use in the process pool') + # This is based on _BackRelay from twister.internal.utils, but modified to # capture both stdout and stderr, without odd stderr handling, and also to # handle stdin @@ -55,8 +56,8 @@ class BackRelayWithInput(protocol.ProcessProtocol): will be called back when the process ends. This C{Deferred} is also associated with the L{_ProcessExecutionError} which C{deferred} fires with earlier in this case so that users can determine when the process - has actually ended, in addition to knowing when bytes have been received - via stderr. + has actually ended, in addition to knowing when bytes have been + received via stderr. """ def __init__(self, deferred, cmd, started_deferred=None, @@ -93,7 +94,7 @@ class BackRelayWithInput(protocol.ProcessProtocol): if self.deferred is not None: stdout, stderr = self.stdout.getvalue(), self.stderr.getvalue() exit_code = reason.value.exitCode - if self.check_exit_code and exit_code <> 0: + if self.check_exit_code and exit_code != 0: self.deferred.errback(self._build_execution_error(exit_code)) else: try: @@ -101,14 +102,15 @@ class BackRelayWithInput(protocol.ProcessProtocol): reason.trap(error.ProcessDone) self.deferred.callback((stdout, stderr)) except: - # NOTE(justinsb): This logic is a little suspicious to me... - # If the callback throws an exception, then errback will be - # called also. However, this is what the unit tests test for... - self.deferred.errback(self._build_execution_error(exit_code)) + # NOTE(justinsb): This logic is a little suspicious to me. + # If the callback throws an exception, then errback will + # be called also. However, this is what the unit tests + # test for. + exec_error = self._build_execution_error(exit_code) + self.deferred.errback(exec_error) elif self.on_process_ended is not None: self.on_process_ended.errback(reason) - def connectionMade(self): if self.started_deferred: self.started_deferred.callback(self) @@ -116,6 +118,7 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.transport.write(str(self.process_input)) self.transport.closeStdin() + def get_process_output(executable, args=None, env=None, path=None, process_reactor=None, check_exit_code=True, process_input=None, started_deferred=None, @@ -142,7 +145,7 @@ def get_process_output(executable, args=None, env=None, path=None, if not args is None: args = [str(x) for x in args] process_reactor.spawnProcess(process_handler, executable, - (executable,)+tuple(args), env, path) + (executable,) + tuple(args), env, path) return deferred @@ -193,9 +196,11 @@ class ProcessPool(object): class SharedPool(object): _instance = None + def __init__(self): if SharedPool._instance is None: self.__class__._instance = ProcessPool() + def __getattr__(self, key): return getattr(self._instance, key) diff --git a/nova/quota.py b/nova/quota.py index 045051207..01dd0ecd4 100644 --- a/nova/quota.py +++ b/nova/quota.py @@ -37,6 +37,7 @@ flags.DEFINE_integer('quota_gigabytes', 1000, flags.DEFINE_integer('quota_floating_ips', 10, 'number of floating ips allowed per project') + def get_quota(context, project_id): rval = {'instances': FLAGS.quota_instances, 'cores': FLAGS.quota_cores, @@ -52,6 +53,7 @@ def get_quota(context, project_id): pass return rval + def allowed_instances(context, num_instances, instance_type): """Check quota and return min(num_instances, allowed_instances)""" project_id = context.project_id @@ -92,4 +94,3 @@ def allowed_floating_ips(context, num_floating_ips): quota = get_quota(context, project_id) allowed_floating_ips = quota['floating_ips'] - used_floating_ips return min(num_floating_ips, allowed_floating_ips) - diff --git a/nova/rpc.py b/nova/rpc.py index 965934205..895820cd0 100644 --- a/nova/rpc.py +++ b/nova/rpc.py @@ -268,6 +268,7 @@ def _unpack_context(msg): LOG.debug('unpacked context: %s', context_dict) return context.RequestContext.from_dict(context_dict) + def _pack_context(msg, context): """Pack context into msg. @@ -280,6 +281,7 @@ def _pack_context(msg, context): for (key, value) in context.to_dict().iteritems()]) msg.update(context) + def call(context, topic, msg): """Sends a message on a topic and wait for a response""" LOG.debug("Making asynchronous call...") diff --git a/nova/server.py b/nova/server.py index c58a15041..cb424caa1 100644 --- a/nova/server.py +++ b/nova/server.py @@ -54,11 +54,11 @@ def stop(pidfile): """ # Get the pid from the pidfile try: - pid = int(open(pidfile,'r').read().strip()) + pid = int(open(pidfile, 'r').read().strip()) except IOError: message = "pidfile %s does not exist. Daemon not running?\n" sys.stderr.write(message % pidfile) - return # not an error in a restart + return # Try killing the daemon process try: @@ -143,6 +143,5 @@ def daemonize(args, name, main): stderr=stderr, uid=FLAGS.uid, gid=FLAGS.gid, - files_preserve=files_to_keep - ): + files_preserve=files_to_keep): main(args) diff --git a/nova/test.py b/nova/test.py index 8e89eb43e..8ef7eca1a 100644 --- a/nova/test.py +++ b/nova/test.py @@ -58,7 +58,7 @@ def skip_if_fake(func): class TrialTestCase(unittest.TestCase): """Test case base class for all unit tests""" - def setUp(self): # pylint: disable-msg=C0103 + def setUp(self): """Run before each test method to initialize test environment""" super(TrialTestCase, self).setUp() # NOTE(vish): We need a better method for creating fixtures for tests @@ -82,8 +82,9 @@ class TrialTestCase(unittest.TestCase): self._monkey_patch_attach() self._original_flags = FLAGS.FlagValuesDict() - def tearDown(self): # pylint: disable-msg=C0103 - """Runs after each test method to finalize/tear down test environment""" + def tearDown(self): + """Runs after each test method to finalize/tear down test + environment.""" try: self.mox.UnsetStubs() self.stubs.UnsetAll() @@ -91,7 +92,8 @@ class TrialTestCase(unittest.TestCase): self.mox.VerifyAll() # NOTE(vish): Clean up any ips associated during the test. ctxt = context.get_admin_context() - db.fixed_ip_disassociate_all_by_timeout(ctxt, FLAGS.host, self.start) + db.fixed_ip_disassociate_all_by_timeout(ctxt, FLAGS.host, + self.start) db.network_disassociate_all(ctxt) rpc.Consumer.attach_to_twisted = self.originalAttach for x in self.injected: @@ -149,6 +151,7 @@ class TrialTestCase(unittest.TestCase): def _monkey_patch_attach(self): self.originalAttach = rpc.Consumer.attach_to_twisted + def _wrapped(innerSelf): rv = self.originalAttach(innerSelf) self.injected.append(rv) @@ -164,7 +167,7 @@ class BaseTestCase(TrialTestCase): DEPRECATED: This is being removed once Tornado is gone, use TrialTestCase. """ - def setUp(self): # pylint: disable-msg=C0103 + def setUp(self): """Run before each test method to initialize test environment""" super(BaseTestCase, self).setUp() # TODO(termie): we could possibly keep a more global registry of @@ -179,7 +182,9 @@ class BaseTestCase(TrialTestCase): """ Push the ioloop along to wait for our test to complete. """ self._waiting = self.ioloop.add_timeout(time.time() + timeout, self._timeout) + def _wait(): + """Wrapped wait function. Called on timeout.""" if self._timed_out: self.fail('test timed out') @@ -198,7 +203,7 @@ class BaseTestCase(TrialTestCase): if self._waiting: try: self.ioloop.remove_timeout(self._waiting) - except Exception: # pylint: disable-msg=W0703 + except Exception: # pylint: disable-msg=W0703 # TODO(jaypipes): This produces a pylint warning. Should # we really be catching Exception and then passing here? pass @@ -219,9 +224,11 @@ class BaseTestCase(TrialTestCase): Example (callback chain, ugly): - d = self.compute.terminate_instance(instance_id) # a Deferred instance + # A deferred instance + d = self.compute.terminate_instance(instance_id) def _describe(_): - d_desc = self.compute.describe_instances() # another Deferred instance + # Another deferred instance + d_desc = self.compute.describe_instances() return d_desc def _checkDescribe(rv): self.assertEqual(rv, []) diff --git a/nova/tests/api/__init__.py b/nova/tests/api/__init__.py index f051e2390..a0681f414 100644 --- a/nova/tests/api/__init__.py +++ b/nova/tests/api/__init__.py @@ -32,10 +32,10 @@ from nova.tests.api.fakes import APIStub class Test(unittest.TestCase): - def setUp(self): # pylint: disable-msg=C0103 + def setUp(self): self.stubs = stubout.StubOutForTesting() - def tearDown(self): # pylint: disable-msg=C0103 + def tearDown(self): self.stubs.UnsetAll() def _request(self, url, subdomain, **kwargs): diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 3f60f38f2..55a7ff32f 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -101,7 +101,7 @@ class XmlConversionTestCase(test.BaseTestCase): class ApiEc2TestCase(test.BaseTestCase): """Unit test for the cloud controller on an EC2 API""" - def setUp(self): # pylint: disable-msg=C0103,C0111 + def setUp(self): super(ApiEc2TestCase, self).setUp() self.manager = manager.AuthManager() diff --git a/nova/tests/compute_unittest.py b/nova/tests/compute_unittest.py index 01e1bcd30..01b5651df 100644 --- a/nova/tests/compute_unittest.py +++ b/nova/tests/compute_unittest.py @@ -37,7 +37,7 @@ FLAGS = flags.FLAGS class ComputeTestCase(test.TrialTestCase): """Test case for compute""" - def setUp(self): # pylint: disable-msg=C0103 + def setUp(self): logging.getLogger().setLevel(logging.DEBUG) super(ComputeTestCase, self).setUp() self.flags(connection_type='fake', @@ -48,7 +48,7 @@ class ComputeTestCase(test.TrialTestCase): self.project = self.manager.create_project('fake', 'fake', 'fake') self.context = context.get_admin_context() - def tearDown(self): # pylint: disable-msg=C0103 + def tearDown(self): self.manager.delete_user(self.user) self.manager.delete_project(self.project) super(ComputeTestCase, self).tearDown() diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index e8dd2624f..44cf47407 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -35,7 +35,7 @@ FLAGS = flags.FLAGS class NetworkTestCase(test.TrialTestCase): """Test cases for network code""" - def setUp(self): # pylint: disable-msg=C0103 + def setUp(self): super(NetworkTestCase, self).setUp() # NOTE(vish): if you change these flags, make sure to change the # flags in the corresponding section in nova-dhcpbridge @@ -65,7 +65,7 @@ class NetworkTestCase(test.TrialTestCase): instance_ref = self._create_instance(1) self.instance2_id = instance_ref['id'] - def tearDown(self): # pylint: disable-msg=C0103 + def tearDown(self): super(NetworkTestCase, self).tearDown() # TODO(termie): this should really be instantiating clean datastores # in between runs, one failure kills all the tests diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index f096ac6fe..d1604c4e5 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -57,7 +57,7 @@ os.makedirs(os.path.join(OSS_TEMPDIR, 'buckets')) class ObjectStoreTestCase(test.TrialTestCase): """Test objectstore API directly.""" - def setUp(self): # pylint: disable-msg=C0103 + def setUp(self): """Setup users and projects.""" super(ObjectStoreTestCase, self).setUp() self.flags(buckets_path=os.path.join(OSS_TEMPDIR, 'buckets'), @@ -73,7 +73,7 @@ class ObjectStoreTestCase(test.TrialTestCase): self.auth_manager.create_project('proj2', 'user2', 'a proj', ['user2']) self.context = context.RequestContext('user1', 'proj1') - def tearDown(self): # pylint: disable-msg=C0103 + def tearDown(self): """Tear down users and projects.""" self.auth_manager.delete_project('proj1') self.auth_manager.delete_project('proj2') @@ -194,7 +194,7 @@ class TestSite(server.Site): class S3APITestCase(test.TrialTestCase): """Test objectstore through S3 API.""" - def setUp(self): # pylint: disable-msg=C0103 + def setUp(self): """Setup users, projects, and start a test server.""" super(S3APITestCase, self).setUp() @@ -309,7 +309,7 @@ class S3APITestCase(test.TrialTestCase): deferred.addCallback(self._ensure_no_buckets) return deferred - def tearDown(self): # pylint: disable-msg=C0103 + def tearDown(self): """Tear down auth and test server.""" self.auth_manager.delete_user('admin') self.auth_manager.delete_project('admin') diff --git a/nova/tests/quota_unittest.py b/nova/tests/quota_unittest.py index 72e44bf52..92f3be508 100644 --- a/nova/tests/quota_unittest.py +++ b/nova/tests/quota_unittest.py @@ -33,7 +33,7 @@ FLAGS = flags.FLAGS class QuotaTestCase(test.TrialTestCase): - def setUp(self): # pylint: disable-msg=C0103 + def setUp(self): logging.getLogger().setLevel(logging.DEBUG) super(QuotaTestCase, self).setUp() self.flags(connection_type='fake', @@ -51,7 +51,7 @@ class QuotaTestCase(test.TrialTestCase): self.context = context.RequestContext(project=self.project, user=self.user) - def tearDown(self): # pylint: disable-msg=C0103 + def tearDown(self): manager.AuthManager().delete_project(self.project) manager.AuthManager().delete_user(self.user) super(QuotaTestCase, self).tearDown() diff --git a/nova/tests/rpc_unittest.py b/nova/tests/rpc_unittest.py index 5d2bb1046..c380393a0 100644 --- a/nova/tests/rpc_unittest.py +++ b/nova/tests/rpc_unittest.py @@ -33,7 +33,7 @@ FLAGS = flags.FLAGS class RpcTestCase(test.TrialTestCase): """Test cases for rpc""" - def setUp(self): # pylint: disable-msg=C0103 + def setUp(self): super(RpcTestCase, self).setUp() self.conn = rpc.Connection.instance() self.receiver = TestReceiver() diff --git a/nova/tests/scheduler_unittest.py b/nova/tests/scheduler_unittest.py index 379f8cdc8..166a32702 100644 --- a/nova/tests/scheduler_unittest.py +++ b/nova/tests/scheduler_unittest.py @@ -44,7 +44,7 @@ class TestDriver(driver.Scheduler): class SchedulerTestCase(test.TrialTestCase): """Test case for scheduler""" - def setUp(self): # pylint: disable=C0103 + def setUp(self): super(SchedulerTestCase, self).setUp() self.flags(scheduler_driver='nova.tests.scheduler_unittest.TestDriver') @@ -73,7 +73,7 @@ class SchedulerTestCase(test.TrialTestCase): class SimpleDriverTestCase(test.TrialTestCase): """Test case for simple driver""" - def setUp(self): # pylint: disable-msg=C0103 + def setUp(self): super(SimpleDriverTestCase, self).setUp() self.flags(connection_type='fake', max_cores=4, @@ -87,7 +87,7 @@ class SimpleDriverTestCase(test.TrialTestCase): self.project = self.manager.create_project('fake', 'fake', 'fake') self.context = context.get_admin_context() - def tearDown(self): # pylint: disable-msg=C0103 + def tearDown(self): self.manager.delete_user(self.user) self.manager.delete_project(self.project) diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index 8533fab0a..6df108a98 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -78,7 +78,7 @@ class ServiceManagerTestCase(test.BaseTestCase): class ServiceTestCase(test.BaseTestCase): """Test cases for Services""" - def setUp(self): # pylint: disable=C0103 + def setUp(self): super(ServiceTestCase, self).setUp() self.mox.StubOutWithMock(service, 'db') self.context = context.get_admin_context() diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 8e2fa11c1..20bae8a7f 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -34,7 +34,7 @@ FLAGS = flags.FLAGS class VolumeTestCase(test.TrialTestCase): """Test Case for volumes""" - def setUp(self): # pylint: disable-msg=C0103 + def setUp(self): logging.getLogger().setLevel(logging.DEBUG) super(VolumeTestCase, self).setUp() self.compute = utils.import_object(FLAGS.compute_manager) diff --git a/nova/twistd.py b/nova/twistd.py index df75b603e..3ec0ff61e 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -53,6 +53,7 @@ class TwistdServerOptions(ServerOptions): class FlagParser(object): # this is a required attribute for gflags syntactic_help = '' + def __init__(self, parser): self.parser = parser @@ -63,6 +64,7 @@ class FlagParser(object): def WrapTwistedOptions(wrapped): class TwistedOptionsToFlags(wrapped): subCommands = None + def __init__(self): # NOTE(termie): _data exists because Twisted stuff expects # to be able to set arbitrary things that are @@ -80,7 +82,8 @@ def WrapTwistedOptions(wrapped): def _absorbFlags(self): twistd_flags = [] - reflect.accumulateClassList(self.__class__, 'optFlags', twistd_flags) + reflect.accumulateClassList(self.__class__, 'optFlags', + twistd_flags) for flag in twistd_flags: key = flag[0].replace('-', '_') if hasattr(FLAGS, key): @@ -89,7 +92,8 @@ def WrapTwistedOptions(wrapped): def _absorbParameters(self): twistd_params = [] - reflect.accumulateClassList(self.__class__, 'optParameters', twistd_params) + reflect.accumulateClassList(self.__class__, 'optParameters', + twistd_params) for param in twistd_params: key = param[0].replace('-', '_') if hasattr(FLAGS, key): @@ -103,13 +107,14 @@ def WrapTwistedOptions(wrapped): def _absorbHandlers(self): twistd_handlers = {} - reflect.addMethodNamesToDict(self.__class__, twistd_handlers, "opt_") + reflect.addMethodNamesToDict(self.__class__, twistd_handlers, + "opt_") # NOTE(termie): Much of the following is derived/copied from # twisted.python.usage with the express purpose of # providing compatibility for name in twistd_handlers.keys(): - method = getattr(self, 'opt_'+name) + method = getattr(self, 'opt_' + name) takesArg = not usage.flagFunction(method, name) doc = getattr(method, '__doc__', None) @@ -125,7 +130,6 @@ def WrapTwistedOptions(wrapped): flags.DEFINE_string(name, None, doc) self._paramHandlers[name] = method - def _doHandlers(self): for flag, handler in self._flagHandlers.iteritems(): if self[flag]: @@ -195,7 +199,7 @@ def stop(pidfile): """ # Get the pid from the pidfile try: - pf = file(pidfile,'r') + pf = file(pidfile, 'r') pid = int(pf.read().strip()) pf.close() except IOError: @@ -204,7 +208,8 @@ def stop(pidfile): if not pid: message = "pidfile %s does not exist. Daemon not running?\n" sys.stderr.write(message % pidfile) - return # not an error in a restart + # Not an error in a restart + return # Try killing the daemon process try: diff --git a/nova/utils.py b/nova/utils.py index 10b27ffec..7683fc9f4 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -39,6 +39,7 @@ from nova.exception import ProcessExecutionError FLAGS = flags.FLAGS TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" + def import_class(import_str): """Returns a class from a string including module and class""" mod_str, _sep, class_str = import_str.rpartition('.') @@ -48,6 +49,7 @@ def import_class(import_str): except (ImportError, ValueError, AttributeError): raise exception.NotFound('Class %s cannot be found' % class_str) + def import_object(import_str): """Returns an object including a module or module and class""" try: @@ -57,6 +59,7 @@ def import_object(import_str): cls = import_class(import_str) return cls() + def fetchfile(url, target): logging.debug("Fetching %s" % url) # c = pycurl.Curl() @@ -68,6 +71,7 @@ def fetchfile(url, target): # fp.close() execute("curl --fail %s -o %s" % (url, target)) + def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): logging.debug("Running cmd: %s", cmd) env = os.environ.copy() @@ -83,7 +87,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): obj.stdin.close() if obj.returncode: logging.debug("Result was %s" % (obj.returncode)) - if check_exit_code and obj.returncode <> 0: + if check_exit_code and obj.returncode != 0: (stdout, stderr) = result raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, @@ -106,7 +110,8 @@ def default_flagfile(filename='nova.conf'): script_dir = os.path.dirname(inspect.stack()[-1][1]) filename = os.path.abspath(os.path.join(script_dir, filename)) if os.path.exists(filename): - sys.argv = sys.argv[:1] + ['--flagfile=%s' % filename] + sys.argv[1:] + flagfile = ['--flagfile=%s' % filename] + sys.argv = sys.argv[:1] + flagfile + sys.argv[1:] def debug(arg): @@ -114,11 +119,11 @@ def debug(arg): return arg -def runthis(prompt, cmd, check_exit_code = True): +def runthis(prompt, cmd, check_exit_code=True): logging.debug("Running %s" % (cmd)) exit_code = subprocess.call(cmd.split(" ")) logging.debug(prompt % (exit_code)) - if check_exit_code and exit_code <> 0: + if check_exit_code and exit_code != 0: raise ProcessExecutionError(exit_code=exit_code, stdout=None, stderr=None, @@ -128,7 +133,7 @@ def runthis(prompt, cmd, check_exit_code = True): def generate_uid(topic, size=8): if topic == "i": # Instances have integer internal ids. - return random.randint(0, 2**32-1) + return random.randint(0, 2 ** 32 - 1) else: characters = '01234567890abcdefghijklmnopqrstuvwxyz' choices = [random.choice(characters) for x in xrange(size)] @@ -136,9 +141,10 @@ def generate_uid(topic, size=8): def generate_mac(): - mac = [0x02, 0x16, 0x3e, random.randint(0x00, 0x7f), - random.randint(0x00, 0xff), random.randint(0x00, 0xff) - ] + mac = [0x02, 0x16, 0x3e, + random.randint(0x00, 0x7f), + random.randint(0x00, 0xff), + random.randint(0x00, 0xff)] return ':'.join(map(lambda x: "%02x" % x, mac)) @@ -201,6 +207,7 @@ class LazyPluggable(object): backend = self.__get_backend() return getattr(backend, key) + def deferredToThread(f): def g(*args, **kwargs): return deferToThread(f, *args, **kwargs) diff --git a/nova/validate.py b/nova/validate.py index 21f4ed286..7ea27daa6 100644 --- a/nova/validate.py +++ b/nova/validate.py @@ -16,18 +16,20 @@ # License for the specific language governing permissions and limitations # under the License. -""" - Decorators for argument validation, courtesy of - http://rmi.net/~lutz/rangetest.html -""" +"""Decorators for argument validation, courtesy of +http://rmi.net/~lutz/rangetest.html""" -def rangetest(**argchecks): # validate ranges for both+defaults - def onDecorator(func): # onCall remembers func and argchecks + +def rangetest(**argchecks): + """Validate ranges for both + defaults""" + + def onDecorator(func): + """onCall remembers func and argchecks""" import sys code = func.__code__ if sys.version_info[0] == 3 else func.func_code - allargs = code.co_varnames[:code.co_argcount] + allargs = code.co_varnames[:code.co_argcount] funcname = func.__name__ - + def onCall(*pargs, **kargs): # all pargs match first N args by position # the rest must be in kargs or omitted defaults @@ -38,7 +40,8 @@ def rangetest(**argchecks): # validate ranges for both+defaults # for all args to be checked if argname in kargs: # was passed by name - if float(kargs[argname]) < low or float(kargs[argname]) > high: + if float(kargs[argname]) < low or \ + float(kargs[argname]) > high: errmsg = '{0} argument "{1}" not in {2}..{3}' errmsg = errmsg.format(funcname, argname, low, high) raise TypeError(errmsg) @@ -46,9 +49,12 @@ def rangetest(**argchecks): # validate ranges for both+defaults elif argname in positionals: # was passed by position position = positionals.index(argname) - if float(pargs[position]) < low or float(pargs[position]) > high: - errmsg = '{0} argument "{1}" with value of {4} not in {2}..{3}' - errmsg = errmsg.format(funcname, argname, low, high, pargs[position]) + if float(pargs[position]) < low or \ + float(pargs[position]) > high: + errmsg = '{0} argument "{1}" with value of {4} ' \ + 'not in {2}..{3}' + errmsg = errmsg.format(funcname, argname, low, high, + pargs[position]) raise TypeError(errmsg) else: pass @@ -62,9 +68,9 @@ def typetest(**argchecks): def onDecorator(func): import sys code = func.__code__ if sys.version_info[0] == 3 else func.func_code - allargs = code.co_varnames[:code.co_argcount] + allargs = code.co_varnames[:code.co_argcount] funcname = func.__name__ - + def onCall(*pargs, **kargs): positionals = list(allargs)[:len(pargs)] for (argname, typeof) in argchecks.items(): @@ -76,12 +82,13 @@ def typetest(**argchecks): elif argname in positionals: position = positionals.index(argname) if not isinstance(pargs[position], typeof): - errmsg = '{0} argument "{1}" with value of {2} not of type {3}' - errmsg = errmsg.format(funcname, argname, pargs[position], typeof) + errmsg = '{0} argument "{1}" with value of {2} ' \ + 'not of type {3}' + errmsg = errmsg.format(funcname, argname, + pargs[position], typeof) raise TypeError(errmsg) else: pass return func(*pargs, **kargs) return onCall return onDecorator - diff --git a/nova/wsgi.py b/nova/wsgi.py index b91d91121..eb305a3d3 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -94,11 +94,11 @@ class Middleware(Application): behavior. """ - def __init__(self, application): # pylint: disable-msg=W0231 + def __init__(self, application): # pylint: disable-msg=W0231 self.application = application @webob.dec.wsgify - def __call__(self, req): # pylint: disable-msg=W0221 + def __call__(self, req): # pylint: disable-msg=W0221 """Override to implement middleware behavior.""" return self.application @@ -216,7 +216,7 @@ class Controller(object): arg_dict['req'] = req result = method(**arg_dict) if type(result) is dict: - return self._serialize(result, req) + return self._serialize(result, req) else: return result @@ -240,6 +240,7 @@ class Controller(object): serializer = Serializer(request.environ, _metadata) return serializer.deserialize(data) + class Serializer(object): """ Serializes and deserializes dictionaries to certain MIME types. @@ -263,12 +264,13 @@ class Serializer(object): elif 'application/xml' in req.accept: self.handler = self._to_xml else: - self.handler = self._to_json # default + # This is the default + self.handler = self._to_json def to_content_type(self, data): """ Serialize a dictionary into a string. - + The format of the string will be decided based on the Content Type requested in self.environ: by Accept: header, or by URL suffix. """ @@ -277,7 +279,7 @@ class Serializer(object): def deserialize(self, datastring): """ Deserialize a string to a dictionary. - + The string must be in the format of a supported MIME type. """ datastring = datastring.strip() @@ -298,7 +300,7 @@ class Serializer(object): def _from_xml_node(self, node, listnames): """ Convert a minidom node to a simple Python type. - + listnames is a collection of names of XML nodes whose subnodes should be considered list items. """ @@ -312,7 +314,8 @@ class Serializer(object): result[attr] = node.attributes[attr].nodeValue for child in node.childNodes: if child.nodeType != node.TEXT_NODE: - result[child.nodeName] = self._from_xml_node(child, listnames) + result[child.nodeName] = self._from_xml_node(child, + listnames) return result def _to_json(self, data): @@ -347,7 +350,8 @@ class Serializer(object): else: node = self._to_xml_node(doc, metadata, k, v) result.appendChild(node) - else: # atom + else: + # Type is atom node = doc.createTextNode(str(data)) result.appendChild(node) return result diff --git a/pylintrc b/pylintrc index 024802835..f07b14980 100644 --- a/pylintrc +++ b/pylintrc @@ -13,7 +13,7 @@ argument-rgx=[a-z_][a-z0-9_]{1,30}$ # Method names should be at least 3 characters long # and be lowecased with underscores -method-rgx=[a-z_][a-z0-9_]{2,50}$ +method-rgx=([a-z_][a-z0-9_]{2,50}|setUp|tearDown)$ # Module names matching nova-* are ok (files in bin/) module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+)|(nova-[a-z0-9_-]+))$ -- cgit From 777b14c27ba9bd5ee298f62323b2170023e44ca6 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 21 Oct 2010 15:26:06 -0700 Subject: PEP8 cleanup in nova/api. There should be no functional changes here, just style changes to get violations down. --- nova/api/__init__.py | 15 ++-- nova/api/ec2/__init__.py | 13 ++-- nova/api/ec2/admin.py | 11 +-- nova/api/ec2/apirequest.py | 8 ++- nova/api/ec2/cloud.py | 107 ++++++++++++---------------- nova/api/ec2/images.py | 6 +- nova/api/ec2/metadatarequesthandler.py | 7 +- nova/api/openstack/__init__.py | 31 ++++---- nova/api/openstack/auth.py | 9 +-- nova/api/openstack/backup_schedules.py | 1 + nova/api/openstack/faults.py | 2 +- nova/api/openstack/flavors.py | 6 +- nova/api/openstack/images.py | 10 ++- nova/api/openstack/ratelimiting/__init__.py | 9 ++- nova/api/openstack/servers.py | 29 ++++---- nova/api/openstack/sharedipgroups.py | 4 +- 16 files changed, 135 insertions(+), 133 deletions(-) diff --git a/nova/api/__init__.py b/nova/api/__init__.py index 8ec7094d7..8a1d9fe32 100644 --- a/nova/api/__init__.py +++ b/nova/api/__init__.py @@ -31,12 +31,13 @@ from nova.api import openstack from nova.api.ec2 import metadatarequesthandler -flags.DEFINE_string('osapi_subdomain', 'api', +flags.DEFINE_string('osapi_subdomain', 'api', 'subdomain running the OpenStack API') -flags.DEFINE_string('ec2api_subdomain', 'ec2', +flags.DEFINE_string('ec2api_subdomain', 'ec2', 'subdomain running the EC2 API') -flags.DEFINE_string('FAKE_subdomain', None, - 'set to api or ec2 to fake the subdomain of the host for testing') +flags.DEFINE_string('FAKE_subdomain', None, + 'set to api or ec2 to fake the subdomain of the host ' + 'for testing') FLAGS = flags.FLAGS @@ -44,7 +45,7 @@ class API(wsgi.Router): """Routes top-level requests to the appropriate controller.""" def __init__(self): - osapidomain = {'sub_domain': [FLAGS.osapi_subdomain]} + osapidomain = {'sub_domain': [FLAGS.osapi_subdomain]} ec2domain = {'sub_domain': [FLAGS.ec2api_subdomain]} # If someone wants to pretend they're hitting the OSAPI subdomain # on their local box, they can set FAKE_subdomain to 'api', which @@ -55,7 +56,7 @@ class API(wsgi.Router): ec2domain = {} mapper = routes.Mapper() mapper.sub_domains = True - mapper.connect("/", controller=self.osapi_versions, + mapper.connect("/", controller=self.osapi_versions, conditions=osapidomain) mapper.connect("/v1.0/{path_info:.*}", controller=openstack.API(), conditions=osapidomain) @@ -107,5 +108,3 @@ class API(wsgi.Router): '2009-04-04', ] return ''.join('%s\n' % v for v in versions) - - diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 5735eb956..0df4d3710 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -62,7 +62,8 @@ class Authenticate(wsgi.Middleware): # Make a copy of args for authentication and signature verification. auth_params = dict(req.params) - auth_params.pop('Signature') # not part of authentication args + # Not part of authentication args + auth_params.pop('Signature') # Authenticate the request. try: @@ -109,9 +110,11 @@ class Router(wsgi.Middleware): 'SignatureVersion', 'Version', 'Timestamp'] args = dict(req.params) try: - action = req.params['Action'] # raise KeyError if omitted + # Raise KeyError if omitted + action = req.params['Action'] for non_arg in non_args: - args.pop(non_arg) # remove, but raise KeyError if omitted + # Remove, but raise KeyError if omitted + args.pop(non_arg) except: raise webob.exc.HTTPBadRequest() @@ -184,7 +187,8 @@ class Authorizer(wsgi.Middleware): context = req.environ['ec2.context'] controller_name = req.environ['ec2.controller'].__class__.__name__ action = req.environ['ec2.action'] - allowed_roles = self.action_roles[controller_name].get(action, ['none']) + allowed_roles = self.action_roles[controller_name].get(action, + ['none']) if self._matches_any_role(context, allowed_roles): return self.application else: @@ -242,4 +246,3 @@ class Executor(wsgi.Application): '%s' '?') % (code, message) return resp - diff --git a/nova/api/ec2/admin.py b/nova/api/ec2/admin.py index 36feae451..1c6ab688d 100644 --- a/nova/api/ec2/admin.py +++ b/nova/api/ec2/admin.py @@ -73,7 +73,7 @@ class AdminController(object): def describe_users(self, _context, **_kwargs): """Returns all users - should be changed to deal with a list.""" return {'userSet': - [user_dict(u) for u in manager.AuthManager().get_users()] } + [user_dict(u) for u in manager.AuthManager().get_users()]} def register_user(self, _context, name, **_kwargs): """Creates a new user, and returns generated credentials.""" @@ -91,7 +91,7 @@ class AdminController(object): def describe_roles(self, context, project_roles=True, **kwargs): """Returns a list of allowed roles.""" roles = manager.AuthManager().get_roles(project_roles) - return { 'roles': [{'role': r} for r in roles]} + return {'roles': [{'role': r} for r in roles]} def describe_user_roles(self, context, user, project=None, **kwargs): """Returns a list of roles for the given user. @@ -99,7 +99,7 @@ class AdminController(object): Specifying project will return only project specific roles. """ roles = manager.AuthManager().get_user_roles(user, project=project) - return { 'roles': [{'role': r} for r in roles]} + return {'roles': [{'role': r} for r in roles]} def modify_user_role(self, context, user, role, project=None, operation='add', **kwargs): @@ -155,9 +155,10 @@ class AdminController(object): 'members': [{'member': m} for m in project.member_ids]} return result - def modify_project_member(self, context, user, project, operation, **kwargs): + def modify_project_member(self, context, user, project, operation, + **kwargs): """Add or remove a user from a project.""" - if operation =='add': + if operation == 'add': manager.AuthManager().add_to_project(user, project) elif operation == 'remove': manager.AuthManager().remove_from_project(user, project) diff --git a/nova/api/ec2/apirequest.py b/nova/api/ec2/apirequest.py index 3c8651582..5758781b6 100644 --- a/nova/api/ec2/apirequest.py +++ b/nova/api/ec2/apirequest.py @@ -44,6 +44,7 @@ def _underscore_to_xmlcase(str): res = _underscore_to_camelcase(str) return res[:1].lower() + res[1:] + def _try_convert(value): """Return a non-string if possible""" if value == 'None': @@ -59,12 +60,12 @@ def _try_convert(value): return value if valueneg[0] == '0': if valueneg[1] in 'xX': - return int(value,16) + return int(value, 16) elif valueneg[1] in 'bB': - return int(value,2) + return int(value, 2) else: try: - return int(value,8) + return int(value, 8) except ValueError: pass try: @@ -80,6 +81,7 @@ def _try_convert(value): except ValueError: return value + class APIRequest(object): def __init__(self, controller, action): self.controller = controller diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 6d4f58499..784697b01 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -48,6 +48,7 @@ flags.DECLARE('storage_availability_zone', 'nova.volume.manager') InvalidInputException = exception.InvalidInputException + class QuotaError(exception.ApiError): """Quota Exceeeded""" pass @@ -137,8 +138,8 @@ class CloudController(object): for node in nodes: rpc.cast(context, '%s.%s' % (FLAGS.compute_topic, node), - { "method": "refresh_security_group", - "args": {"security_group_id": security_group.id}}) + {"method": "refresh_security_group", + "args": {"security_group_id": security_group.id}}) def get_metadata(self, address): ctxt = context.get_admin_context() @@ -147,48 +148,42 @@ class CloudController(object): return None mpi = self._get_mpi_data(ctxt, instance_ref['project_id']) if instance_ref['key_name']: - keys = { - '0': { - '_name': instance_ref['key_name'], - 'openssh-key': instance_ref['key_data'] - } - } + keys = {'0': {'_name': instance_ref['key_name'], + 'openssh-key': instance_ref['key_data']}} else: keys = '' hostname = instance_ref['hostname'] floating_ip = db.instance_get_floating_address(ctxt, instance_ref['id']) + ec2_id = internal_id_to_ec2_id(instance_ref['internal_id']) data = { 'user-data': base64.b64decode(instance_ref['user_data']), 'meta-data': { 'ami-id': instance_ref['image_id'], 'ami-launch-index': instance_ref['launch_index'], 'ami-manifest-path': 'FIXME', - 'block-device-mapping': { # TODO(vish): replace with real data + 'block-device-mapping': { + # TODO(vish): replace with real data 'ami': 'sda1', 'ephemeral0': 'sda2', 'root': '/dev/sda1', - 'swap': 'sda3' - }, + 'swap': 'sda3'}, 'hostname': hostname, 'instance-action': 'none', - 'instance-id': internal_id_to_ec2_id(instance_ref['internal_id']), + 'instance-id': ec2_id, 'instance-type': instance_ref['instance_type'], 'local-hostname': hostname, 'local-ipv4': address, 'kernel-id': instance_ref['kernel_id'], - 'placement': { - 'availability-zone': 'nova' # TODO(vish): real zone - }, + # TODO(vish): real zone + 'placement': {'availability-zone': 'nova'}, 'public-hostname': hostname, 'public-ipv4': floating_ip or '', 'public-keys': keys, 'ramdisk-id': instance_ref['ramdisk_id'], 'reservation-id': instance_ref['reservation_id'], 'security-groups': '', - 'mpi': mpi - } - } + 'mpi': mpi}} if False: # TODO(vish): store ancestor ids data['ancestor-ami-ids'] = [] if False: # TODO(vish): store product codes @@ -211,7 +206,7 @@ class CloudController(object): 'regionEndpoint': FLAGS.ec2_url}] if region_name: regions = [r for r in regions if r['regionName'] in region_name] - return {'regionInfo': regions } + return {'regionInfo': regions} def describe_snapshots(self, context, @@ -237,7 +232,8 @@ class CloudController(object): for key_pair in key_pairs: # filter out the vpn keys suffix = FLAGS.vpn_key_suffix - if context.user.is_admin() or not key_pair['name'].endswith(suffix): + if context.user.is_admin() or \ + not key_pair['name'].endswith(suffix): result.append({ 'keyName': key_pair['name'], 'keyFingerprint': key_pair['fingerprint'], @@ -271,7 +267,7 @@ class CloudController(object): if not group_name is None: groups = [g for g in groups if g.name in group_name] - return {'securityGroupInfo': groups } + return {'securityGroupInfo': groups} def _format_security_group(self, context, group): g = {} @@ -295,13 +291,10 @@ class CloudController(object): g['ipPermissions'] += [r] return g - - def _authorize_revoke_rule_args_to_dict(self, context, - to_port=None, from_port=None, - ip_protocol=None, cidr_ip=None, - user_id=None, - source_security_group_name=None, - source_security_group_owner_id=None): + def _revoke_rule_args_to_dict(self, context, to_port=None, from_port=None, + ip_protocol=None, cidr_ip=None, user_id=None, + source_security_group_name=None, + source_security_group_owner_id=None): values = {} @@ -322,16 +315,16 @@ class CloudController(object): values['cidr'] = '0.0.0.0/0' if ip_protocol and from_port and to_port: - from_port = int(from_port) - to_port = int(to_port) + from_port = int(from_port) + to_port = int(to_port) ip_protocol = str(ip_protocol) - if ip_protocol.upper() not in ['TCP','UDP','ICMP']: - raise InvalidInputException('%s is not a valid ipProtocol' % - (ip_protocol,)) + if ip_protocol.upper() not in ['TCP', 'UDP', 'ICMP']: + raise InvalidInputException('%s is not a valid ipProtocol' % + (ip_protocol,)) if ((min(from_port, to_port) < -1) or (max(from_port, to_port) > 65535)): - raise InvalidInputException('Invalid port range') + raise InvalidInputException('Invalid port range') values['protocol'] = ip_protocol values['from_port'] = from_port @@ -343,7 +336,6 @@ class CloudController(object): return values - def _security_group_rule_exists(self, security_group, values): """Indicates whether the specified rule values are already defined in the given security group. @@ -362,20 +354,19 @@ class CloudController(object): return True return False - def revoke_security_group_ingress(self, context, group_name, **kwargs): self._ensure_default_security_group(context) security_group = db.security_group_get_by_name(context, context.project_id, group_name) - criteria = self._authorize_revoke_rule_args_to_dict(context, **kwargs) + criteria = self._revoke_rule_args_to_dict(context, **kwargs) if criteria == None: raise exception.ApiError("No rule for the specified parameters.") for rule in security_group.rules: match = True - for (k,v) in criteria.iteritems(): + for (k, v) in criteria.iteritems(): if getattr(rule, k, False) != v: match = False if match: @@ -394,7 +385,7 @@ class CloudController(object): context.project_id, group_name) - values = self._authorize_revoke_rule_args_to_dict(context, **kwargs) + values = self._revoke_rule_args_to_dict(context, **kwargs) values['parent_group_id'] = security_group.id if self._security_group_rule_exists(security_group, values): @@ -407,7 +398,6 @@ class CloudController(object): return True - def _get_source_project_id(self, context, source_security_group_owner_id): if source_security_group_owner_id: # Parse user:project for source group. @@ -425,13 +415,12 @@ class CloudController(object): return source_project_id - def create_security_group(self, context, group_name, group_description): self._ensure_default_security_group(context) if db.security_group_exists(context, context.project_id, group_name): raise exception.ApiError('group %s already exists' % group_name) - group = {'user_id' : context.user.id, + group = {'user_id': context.user.id, 'project_id': context.project_id, 'name': group_name, 'description': group_description} @@ -440,7 +429,6 @@ class CloudController(object): return {'securityGroupSet': [self._format_security_group(context, group_ref)]} - def delete_security_group(self, context, group_name, **kwargs): security_group = db.security_group_get_by_name(context, context.project_id, @@ -448,7 +436,6 @@ class CloudController(object): db.security_group_destroy(context, security_group.id) return True - def get_console_output(self, context, instance_id, **kwargs): # instance_id is passed in as a list of instances ec2_id = instance_id[0] @@ -457,13 +444,13 @@ class CloudController(object): output = rpc.call(context, '%s.%s' % (FLAGS.compute_topic, instance_ref['host']), - {"method" : "get_console_output", - "args" : {"instance_id": instance_ref['id']}}) + {"method": "get_console_output", + "args": {"instance_id": instance_ref['id']}}) now = datetime.datetime.utcnow() - return { "InstanceId" : ec2_id, - "Timestamp" : now, - "output" : base64.b64encode(output) } + return {"InstanceId": ec2_id, + "Timestamp": now, + "output": base64.b64encode(output)} def describe_volumes(self, context, **kwargs): if context.user.is_admin(): @@ -530,7 +517,6 @@ class CloudController(object): return {'volumeSet': [self._format_volume(context, volume_ref)]} - def attach_volume(self, context, volume_id, instance_id, device, **kwargs): volume_ref = db.volume_get_by_ec2_id(context, volume_id) # TODO(vish): abstract status checking? @@ -633,8 +619,7 @@ class CloudController(object): i['imageId'] = instance['image_id'] i['instanceState'] = { 'code': instance['state'], - 'name': instance['state_description'] - } + 'name': instance['state_description']} fixed_addr = None floating_addr = None if instance['fixed_ip']: @@ -656,7 +641,7 @@ class CloudController(object): i['amiLaunchIndex'] = instance['launch_index'] i['displayName'] = instance['display_name'] i['displayDescription'] = instance['display_description'] - if not reservations.has_key(instance['reservation_id']): + if instance['reservation_id'] not in reservations: r = {} r['reservationId'] = instance['reservation_id'] r['ownerId'] = instance['project_id'] @@ -757,10 +742,10 @@ class CloudController(object): context.project_id, 'default') except exception.NotFound: - values = { 'name' : 'default', - 'description' : 'default', - 'user_id' : context.user.id, - 'project_id' : context.project_id } + values = {'name': 'default', + 'description': 'default', + 'user_id': context.user.id, + 'project_id': context.project_id} group = db.security_group_create(context, values) def run_instances(self, context, **kwargs): @@ -804,7 +789,7 @@ class CloudController(object): logging.debug("Going to run %s instances...", num_instances) launch_time = time.strftime('%Y-%m-%dT%H:%M:%SZ', time.gmtime()) key_data = None - if kwargs.has_key('key_name'): + if 'key_name' in kwargs: key_pair_ref = db.key_pair_get(context, context.user.id, kwargs['key_name']) @@ -883,7 +868,6 @@ class CloudController(object): (context.project.name, context.user.name, inst_id)) return self._format_run_instances(context, reservation_id) - def terminate_instances(self, context, instance_id, **kwargs): """Terminate each instance in instance_id, which is a list of ec2 ids. @@ -991,7 +975,7 @@ class CloudController(object): def register_image(self, context, image_location=None, **kwargs): # FIXME: should the objectstore be doing these authorization checks? - if image_location is None and kwargs.has_key('name'): + if image_location is None and 'name' in kwargs: image_location = kwargs['name'] image_id = images.register(context, image_location) logging.debug("Registered %s as %s" % (image_location, image_id)) @@ -1009,7 +993,8 @@ class CloudController(object): result['launchPermission'].append({'group': 'all'}) return result - def modify_image_attribute(self, context, image_id, attribute, operation_type, **kwargs): + def modify_image_attribute(self, context, image_id, attribute, + operation_type, **kwargs): # TODO(devcamcar): Support users and groups other than 'all'. if attribute != 'launchPermission': raise exception.ApiError('attribute not supported: %s' % attribute) diff --git a/nova/api/ec2/images.py b/nova/api/ec2/images.py index f0a43dad6..60f9008e9 100644 --- a/nova/api/ec2/images.py +++ b/nova/api/ec2/images.py @@ -43,6 +43,7 @@ def modify(context, image_id, operation): return True + def update(context, image_id, attributes): """update an image's attributes / info.json""" attributes.update({"image_id": image_id}) @@ -52,6 +53,7 @@ def update(context, image_id, attributes): query_args=qs(attributes)) return True + def register(context, image_location): """ rpc call to register a new image based from a manifest """ @@ -64,13 +66,14 @@ def register(context, image_location): return image_id + def list(context, filter_list=[]): """ return a list of all images that a user can see optionally filtered by a list of image_id """ if FLAGS.connection_type == 'fake': - return [{ 'imageId' : 'bar'}] + return [{'imageId': 'bar'}] # FIXME: send along the list of only_images to check for response = conn(context).make_request( @@ -82,6 +85,7 @@ def list(context, filter_list=[]): return [i for i in result if i['imageId'] in filter_list] return result + def get(context, image_id): """return a image object if the context has permissions""" result = list(context, [image_id]) diff --git a/nova/api/ec2/metadatarequesthandler.py b/nova/api/ec2/metadatarequesthandler.py index 08a8040ca..2f4f414cc 100644 --- a/nova/api/ec2/metadatarequesthandler.py +++ b/nova/api/ec2/metadatarequesthandler.py @@ -27,7 +27,6 @@ from nova.api.ec2 import cloud class MetadataRequestHandler(object): - """Serve metadata from the EC2 API.""" def print_data(self, data): @@ -43,7 +42,8 @@ class MetadataRequestHandler(object): else: output += '/' output += '\n' - return output[:-1] # cut off last \n + # Cut off last \n + return output[:-1] elif isinstance(data, list): return '\n'.join(data) else: @@ -65,7 +65,8 @@ class MetadataRequestHandler(object): cc = cloud.CloudController() meta_data = cc.get_metadata(req.remote_addr) if meta_data is None: - logging.error('Failed to get metadata for ip: %s' % req.remote_addr) + logging.error('Failed to get metadata for ip: %s' % + req.remote_addr) raise webob.exc.HTTPNotFound() data = self.lookup(req.path_info, meta_data) if data is None: diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 5706dbc09..bb86f08dd 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -43,9 +43,10 @@ from nova.auth import manager FLAGS = flags.FLAGS flags.DEFINE_string('nova_api_auth', - 'nova.api.openstack.auth.BasicApiAuthManager', + 'nova.api.openstack.auth.BasicApiAuthManager', 'The auth mechanism to use for the OpenStack API implemenation') + class API(wsgi.Middleware): """WSGI entry point for all OpenStack API requests.""" @@ -53,6 +54,7 @@ class API(wsgi.Middleware): app = AuthMiddleware(RateLimitingMiddleware(APIRouter())) super(API, self).__init__(app) + class AuthMiddleware(wsgi.Middleware): """Authorize the openstack API request or return an HTTP Forbidden.""" @@ -62,7 +64,7 @@ class AuthMiddleware(wsgi.Middleware): @webob.dec.wsgify def __call__(self, req): - if not req.headers.has_key("X-Auth-Token"): + if 'X-Auth-Token' not in req.headers: return self.auth_driver.authenticate(req) user = self.auth_driver.authorize_token(req.headers["X-Auth-Token"]) @@ -70,11 +72,12 @@ class AuthMiddleware(wsgi.Middleware): if not user: return faults.Fault(webob.exc.HTTPUnauthorized()) - if not req.environ.has_key('nova.context'): + if 'nova.context' not in req.environ: req.environ['nova.context'] = {} req.environ['nova.context']['user'] = user return self.application + class RateLimitingMiddleware(wsgi.Middleware): """Rate limit incoming requests according to the OpenStack rate limits.""" @@ -87,7 +90,7 @@ class RateLimitingMiddleware(wsgi.Middleware): """ super(RateLimitingMiddleware, self).__init__(application) if not service_host: - #TODO(gundlach): These limits were based on limitations of Cloud + #TODO(gundlach): These limits were based on limitations of Cloud #Servers. We should revisit them in Nova. self.limiter = ratelimiting.Limiter(limits={ 'DELETE': (100, ratelimiting.PER_MINUTE), @@ -102,13 +105,14 @@ class RateLimitingMiddleware(wsgi.Middleware): @webob.dec.wsgify def __call__(self, req): """Rate limit the request. - - If the request should be rate limited, return a 413 status with a + + If the request should be rate limited, return a 413 status with a Retry-After header giving the time when the request would succeed. """ user_id = req.environ['nova.context']['user']['id'] action_name = self.get_action_name(req) - if not action_name: # not rate limited + if not action_name: + # Not rate limited return self.application delay = self.get_delay(action_name, user_id) if delay: @@ -152,13 +156,13 @@ class APIRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() mapper.resource("server", "servers", controller=servers.Controller(), - collection={ 'detail': 'GET'}, - member={'action':'POST'}) + collection={'detail': 'GET'}, + member={'action': 'POST'}) - mapper.resource("backup_schedule", "backup_schedules", + mapper.resource("backup_schedule", "backup_schedules", controller=backup_schedules.Controller(), - parent_resource=dict(member_name='server', - collection_name = 'servers')) + parent_resource=dict(member_name='server', + collection_name='servers')) mapper.resource("image", "images", controller=images.Controller(), collection={'detail': 'GET'}) @@ -172,7 +176,7 @@ class APIRouter(wsgi.Router): def limited(items, req): """Return a slice of items according to requested offset and limit. - + items - a sliceable req - wobob.Request possibly containing offset and limit GET variables. offset is where to start in the list, and limit is the maximum number @@ -187,4 +191,3 @@ def limited(items, req): limit = min(1000, limit) range_end = offset + limit return items[offset:range_end] - diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 7aba55728..ff428ff70 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -15,9 +15,11 @@ from nova.api.openstack import faults FLAGS = flags.FLAGS + class Context(object): pass + class BasicApiAuthManager(object): """ Implements a somewhat rudimentary version of OpenStack Auth""" @@ -61,7 +63,7 @@ class BasicApiAuthManager(object): def authorize_token(self, token_hash): """ retrieves user information from the datastore given a token - + If the token has expired, returns None If the token is not found, returns None Otherwise returns dict(id=(the authorized user's id)) @@ -69,7 +71,7 @@ class BasicApiAuthManager(object): This method will also remove the token if the timestamp is older than 2 days ago. """ - token = self.db.auth_get_token(self.context, token_hash) + token = self.db.auth_get_token(self.context, token_hash) if token: delta = datetime.datetime.now() - token.created_at if delta.days >= 2: @@ -94,8 +96,7 @@ class BasicApiAuthManager(object): token_dict['user_id'] = user.id token = self.db.auth_create_token(self.context, token_dict) return token, user - return None, None + return None, None def _get_server_mgmt_url(self): return 'https://%s/v1.0/' % self.host - diff --git a/nova/api/openstack/backup_schedules.py b/nova/api/openstack/backup_schedules.py index db240c65a..3ed691d7b 100644 --- a/nova/api/openstack/backup_schedules.py +++ b/nova/api/openstack/backup_schedules.py @@ -22,6 +22,7 @@ from nova import wsgi from nova.api.openstack import faults import nova.image.service + class Controller(wsgi.Controller): def __init__(self): pass diff --git a/nova/api/openstack/faults.py b/nova/api/openstack/faults.py index 32e5c866f..e69e51439 100644 --- a/nova/api/openstack/faults.py +++ b/nova/api/openstack/faults.py @@ -55,7 +55,7 @@ class Fault(webob.exc.HTTPException): if code == 413: retry = self.wrapped_exc.headers['Retry-After'] fault_data[fault_name]['retryAfter'] = retry - # 'code' is an attribute on the fault tag itself + # 'code' is an attribute on the fault tag itself metadata = {'application/xml': {'attributes': {fault_name: 'code'}}} serializer = wsgi.Serializer(req.environ, metadata) self.wrapped_exc.body = serializer.to_content_type(fault_data) diff --git a/nova/api/openstack/flavors.py b/nova/api/openstack/flavors.py index 793984a5d..f23f74fd1 100644 --- a/nova/api/openstack/flavors.py +++ b/nova/api/openstack/flavors.py @@ -22,16 +22,14 @@ from nova.compute import instance_types from nova import wsgi import nova.api.openstack + class Controller(wsgi.Controller): """Flavor controller for the OpenStack API.""" _serialization_metadata = { 'application/xml': { "attributes": { - "flavor": [ "id", "name", "ram", "disk" ] - } - } - } + "flavor": ["id", "name", "ram", "disk"]}}} def index(self, req): """Return all flavors in brief.""" diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index 5ccf659f7..5bc915e63 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -27,16 +27,14 @@ from nova.api.openstack import faults FLAGS = flags.FLAGS + class Controller(wsgi.Controller): _serialization_metadata = { 'application/xml': { "attributes": { - "image": [ "id", "name", "updated", "created", "status", - "serverId", "progress" ] - } - } - } + "image": ["id", "name", "updated", "created", "status", + "serverId", "progress"]}}} def __init__(self): self._service = utils.import_object(FLAGS.image_service) @@ -72,6 +70,6 @@ class Controller(wsgi.Controller): raise faults.Fault(exc.HTTPNotFound()) def update(self, req, id): - # Users may not modify public images, and that's all that + # Users may not modify public images, and that's all that # we support for now. raise faults.Fault(exc.HTTPNotFound()) diff --git a/nova/api/openstack/ratelimiting/__init__.py b/nova/api/openstack/ratelimiting/__init__.py index f843bac0f..9e028ecf5 100644 --- a/nova/api/openstack/ratelimiting/__init__.py +++ b/nova/api/openstack/ratelimiting/__init__.py @@ -13,6 +13,7 @@ PER_MINUTE = 60 PER_HOUR = 60 * 60 PER_DAY = 60 * 60 * 24 + class Limiter(object): """Class providing rate limiting of arbitrary actions.""" @@ -101,7 +102,8 @@ class WSGIApp(object): return webob.exc.HTTPForbidden( headers={'X-Wait-Seconds': "%.2f" % delay}) else: - return '' # 200 OK + # 200 OK + return '' class WSGIAppProxy(object): @@ -109,7 +111,7 @@ class WSGIAppProxy(object): """Limiter lookalike that proxies to a ratelimiting.WSGIApp.""" def __init__(self, service_host): - """Creates a proxy pointing to a ratelimiting.WSGIApp at the given + """Creates a proxy pointing to a ratelimiting.WSGIApp at the given host.""" self.service_host = service_host @@ -118,5 +120,6 @@ class WSGIAppProxy(object): conn.request('POST', '/limiter/%s/%s' % (username, action)) resp = conn.getresponse() if resp.status == 200: - return None # no delay + # No delay + return None return float(resp.getheader('X-Wait-Seconds')) diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index a73591ccc..ef773c3be 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -34,30 +34,32 @@ import nova.image.service FLAGS = flags.FLAGS + def _filter_params(inst_dict): """ Extracts all updatable parameters for a server update request """ keys = dict(name='name', admin_pass='adminPass') new_attrs = {} for k, v in keys.items(): - if inst_dict.has_key(v): + if v in inst_dict: new_attrs[k] = inst_dict[v] return new_attrs + def _entity_list(entities): """ Coerces a list of servers into proper dictionary format """ return dict(servers=entities) + def _entity_detail(inst): """ Maps everything to Rackspace-like attributes for return""" power_mapping = { - power_state.NOSTATE: 'build', - power_state.RUNNING: 'active', - power_state.BLOCKED: 'active', - power_state.PAUSED: 'suspended', + power_state.NOSTATE: 'build', + power_state.RUNNING: 'active', + power_state.BLOCKED: 'active', + power_state.PAUSED: 'suspended', power_state.SHUTDOWN: 'active', - power_state.SHUTOFF: 'active', - power_state.CRASHED: 'error' - } + power_state.SHUTOFF: 'active', + power_state.CRASHED: 'error'} inst_dict = {} mapped_keys = dict(status='state', imageId='image_id', @@ -73,21 +75,20 @@ def _entity_detail(inst): return dict(server=inst_dict) + def _entity_inst(inst): """ Filters all model attributes save for id and name """ return dict(server=dict(id=inst['id'], name=inst['server_name'])) + class Controller(wsgi.Controller): """ The Server API controller for the OpenStack API """ _serialization_metadata = { 'application/xml': { "attributes": { - "server": [ "id", "imageId", "name", "flavorId", "hostId", - "status", "progress", "progress" ] - } - } - } + "server": ["id", "imageId", "name", "flavorId", "hostId", + "status", "progress", "progress"]}}} def __init__(self, db_driver=None): if not db_driver: @@ -209,7 +210,7 @@ class Controller(wsgi.Controller): image = img_service.show(image_id) if not image: - raise Exception, "Image not found" + raise Exception("Image not found") inst['server_name'] = env['server']['name'] inst['image_id'] = image_id diff --git a/nova/api/openstack/sharedipgroups.py b/nova/api/openstack/sharedipgroups.py index 4d2d0ede1..e805ca9f7 100644 --- a/nova/api/openstack/sharedipgroups.py +++ b/nova/api/openstack/sharedipgroups.py @@ -17,4 +17,6 @@ from nova import wsgi -class Controller(wsgi.Controller): pass + +class Controller(wsgi.Controller): + pass -- cgit From 7af25f1476f6a30cb45b4a1a990efecc9239169e Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 21 Oct 2010 16:15:26 -0700 Subject: PEP8 cleanup in nova/db. There should be no functional changes here, just style changes to get violations down. --- nova/db/api.py | 12 +- nova/db/sqlalchemy/api.py | 735 +++++++++++++++++++++--------------------- nova/db/sqlalchemy/models.py | 44 +-- nova/db/sqlalchemy/session.py | 3 +- 4 files changed, 408 insertions(+), 386 deletions(-) diff --git a/nova/db/api.py b/nova/db/api.py index 6dbf3b809..0731e2e05 100644 --- a/nova/db/api.py +++ b/nova/db/api.py @@ -256,10 +256,12 @@ def instance_get_all(context): """Get all instances.""" return IMPL.instance_get_all(context) + def instance_get_all_by_user(context, user_id): """Get all instances.""" return IMPL.instance_get_all_by_user(context, user_id) + def instance_get_all_by_project(context, project_id): """Get all instance belonging to a project.""" return IMPL.instance_get_all_by_project(context, project_id) @@ -306,7 +308,8 @@ def instance_update(context, instance_id, values): def instance_add_security_group(context, instance_id, security_group_id): """Associate the given security group with the given instance""" - return IMPL.instance_add_security_group(context, instance_id, security_group_id) + return IMPL.instance_add_security_group(context, instance_id, + security_group_id) ################### @@ -482,10 +485,12 @@ def auth_destroy_token(context, token): """Destroy an auth token""" return IMPL.auth_destroy_token(context, token) + def auth_get_token(context, token_hash): """Retrieves a token given the hash representing it""" return IMPL.auth_get_token(context, token_hash) + def auth_create_token(context, token): """Creates a new token""" return IMPL.auth_create_token(context, token) @@ -644,7 +649,9 @@ def security_group_rule_create(context, values): def security_group_rule_get_by_security_group(context, security_group_id): """Get all rules for a a given security group""" - return IMPL.security_group_rule_get_by_security_group(context, security_group_id) + return IMPL.security_group_rule_get_by_security_group(context, + security_group_id) + def security_group_rule_destroy(context, security_group_rule_id): """Deletes a security group rule""" @@ -767,4 +774,3 @@ def host_get_networks(context, host): network host """ return IMPL.host_get_networks(context, host) - diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 209d6e51f..0cbe56499 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -124,10 +124,10 @@ def service_get(context, service_id, session=None): if not session: session = get_session() - result = session.query(models.Service - ).filter_by(id=service_id - ).filter_by(deleted=can_read_deleted(context) - ).first() + result = session.query(models.Service).\ + filter_by(id=service_id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() if not result: raise exception.NotFound('No service for id %s' % service_id) @@ -138,23 +138,23 @@ def service_get(context, service_id, session=None): @require_admin_context def service_get_all_by_topic(context, topic): session = get_session() - return session.query(models.Service - ).filter_by(deleted=False - ).filter_by(disabled=False - ).filter_by(topic=topic - ).all() + return session.query(models.Service).\ + filter_by(deleted=False).\ + filter_by(disabled=False).\ + filter_by(topic=topic).\ + all() @require_admin_context def _service_get_all_topic_subquery(context, session, topic, subq, label): sort_value = getattr(subq.c, label) - return session.query(models.Service, func.coalesce(sort_value, 0) - ).filter_by(topic=topic - ).filter_by(deleted=False - ).filter_by(disabled=False - ).outerjoin((subq, models.Service.host == subq.c.host) - ).order_by(sort_value - ).all() + return session.query(models.Service, func.coalesce(sort_value, 0)).\ + filter_by(topic=topic).\ + filter_by(deleted=False).\ + filter_by(disabled=False).\ + outerjoin((subq, models.Service.host == subq.c.host)).\ + order_by(sort_value).\ + all() @require_admin_context @@ -171,10 +171,10 @@ def service_get_all_compute_sorted(context): topic = 'compute' label = 'instance_cores' subq = session.query(models.Instance.host, - func.sum(models.Instance.vcpus).label(label) - ).filter_by(deleted=False - ).group_by(models.Instance.host - ).subquery() + func.sum(models.Instance.vcpus).label(label)).\ + filter_by(deleted=False).\ + group_by(models.Instance.host).\ + subquery() return _service_get_all_topic_subquery(context, session, topic, @@ -189,10 +189,10 @@ def service_get_all_network_sorted(context): topic = 'network' label = 'network_count' subq = session.query(models.Network.host, - func.count(models.Network.id).label(label) - ).filter_by(deleted=False - ).group_by(models.Network.host - ).subquery() + func.count(models.Network.id).label(label)).\ + filter_by(deleted=False).\ + group_by(models.Network.host).\ + subquery() return _service_get_all_topic_subquery(context, session, topic, @@ -207,10 +207,10 @@ def service_get_all_volume_sorted(context): topic = 'volume' label = 'volume_gigabytes' subq = session.query(models.Volume.host, - func.sum(models.Volume.size).label(label) - ).filter_by(deleted=False - ).group_by(models.Volume.host - ).subquery() + func.sum(models.Volume.size).label(label)).\ + filter_by(deleted=False).\ + group_by(models.Volume.host).\ + subquery() return _service_get_all_topic_subquery(context, session, topic, @@ -221,11 +221,11 @@ def service_get_all_volume_sorted(context): @require_admin_context def service_get_by_args(context, host, binary): session = get_session() - result = session.query(models.Service - ).filter_by(host=host - ).filter_by(binary=binary - ).filter_by(deleted=can_read_deleted(context) - ).first() + result = session.query(models.Service).\ + filter_by(host=host).\ + filter_by(binary=binary).\ + filter_by(deleted=can_read_deleted(context)).\ + first() if not result: raise exception.NotFound('No service for %s, %s' % (host, binary)) @@ -259,13 +259,13 @@ def floating_ip_allocate_address(context, host, project_id): authorize_project_context(context, project_id) session = get_session() with session.begin(): - floating_ip_ref = session.query(models.FloatingIp - ).filter_by(host=host - ).filter_by(fixed_ip_id=None - ).filter_by(project_id=None - ).filter_by(deleted=False - ).with_lockmode('update' - ).first() + floating_ip_ref = session.query(models.FloatingIp).\ + filter_by(host=host).\ + filter_by(fixed_ip_id=None).\ + filter_by(project_id=None).\ + filter_by(deleted=False).\ + with_lockmode('update').\ + first() # NOTE(vish): if with_lockmode isn't supported, as in sqlite, # then this has concurrency issues if not floating_ip_ref: @@ -288,10 +288,10 @@ def floating_ip_create(context, values): def floating_ip_count_by_project(context, project_id): authorize_project_context(context, project_id) session = get_session() - return session.query(models.FloatingIp - ).filter_by(project_id=project_id - ).filter_by(deleted=False - ).count() + return session.query(models.FloatingIp).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + count() @require_context @@ -354,31 +354,31 @@ def floating_ip_disassociate(context, address): @require_admin_context def floating_ip_get_all(context): session = get_session() - return session.query(models.FloatingIp - ).options(joinedload_all('fixed_ip.instance') - ).filter_by(deleted=False - ).all() + return session.query(models.FloatingIp).\ + options(joinedload_all('fixed_ip.instance')).\ + filter_by(deleted=False).\ + all() @require_admin_context def floating_ip_get_all_by_host(context, host): session = get_session() - return session.query(models.FloatingIp - ).options(joinedload_all('fixed_ip.instance') - ).filter_by(host=host - ).filter_by(deleted=False - ).all() + return session.query(models.FloatingIp).\ + options(joinedload_all('fixed_ip.instance')).\ + filter_by(host=host).\ + filter_by(deleted=False).\ + all() @require_context def floating_ip_get_all_by_project(context, project_id): authorize_project_context(context, project_id) session = get_session() - return session.query(models.FloatingIp - ).options(joinedload_all('fixed_ip.instance') - ).filter_by(project_id=project_id - ).filter_by(deleted=False - ).all() + return session.query(models.FloatingIp).\ + options(joinedload_all('fixed_ip.instance')).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + all() @require_context @@ -387,10 +387,10 @@ def floating_ip_get_by_address(context, address, session=None): if not session: session = get_session() - result = session.query(models.FloatingIp - ).filter_by(address=address - ).filter_by(deleted=can_read_deleted(context) - ).first() + result = session.query(models.FloatingIp).\ + filter_by(address=address).\ + filter_by(deleted=can_read_deleted(context)).\ + first() if not result: raise exception.NotFound('No fixed ip for address %s' % address) @@ -405,12 +405,12 @@ def fixed_ip_associate(context, address, instance_id): session = get_session() with session.begin(): instance = instance_get(context, instance_id, session=session) - fixed_ip_ref = session.query(models.FixedIp - ).filter_by(address=address - ).filter_by(deleted=False - ).filter_by(instance=None - ).with_lockmode('update' - ).first() + fixed_ip_ref = session.query(models.FixedIp).\ + filter_by(address=address).\ + filter_by(deleted=False).\ + filter_by(instance=None).\ + with_lockmode('update').\ + first() # NOTE(vish): if with_lockmode isn't supported, as in sqlite, # then this has concurrency issues if not fixed_ip_ref: @@ -425,13 +425,13 @@ def fixed_ip_associate_pool(context, network_id, instance_id): with session.begin(): network_or_none = or_(models.FixedIp.network_id == network_id, models.FixedIp.network_id == None) - fixed_ip_ref = session.query(models.FixedIp - ).filter(network_or_none - ).filter_by(reserved=False - ).filter_by(deleted=False - ).filter_by(instance=None - ).with_lockmode('update' - ).first() + fixed_ip_ref = session.query(models.FixedIp).\ + filter(network_or_none).\ + filter_by(reserved=False).\ + filter_by(deleted=False).\ + filter_by(instance=None).\ + with_lockmode('update').\ + first() # NOTE(vish): if with_lockmode isn't supported, as in sqlite, # then this has concurrency issues if not fixed_ip_ref: @@ -455,6 +455,7 @@ def fixed_ip_create(_context, values): fixed_ip_ref.save() return fixed_ip_ref['address'] + @require_context def fixed_ip_disassociate(context, address): session = get_session() @@ -465,6 +466,7 @@ def fixed_ip_disassociate(context, address): fixed_ip_ref.instance = None fixed_ip_ref.save(session=session) + @require_admin_context def fixed_ip_disassociate_all_by_timeout(_context, host, time): session = get_session() @@ -486,12 +488,12 @@ def fixed_ip_disassociate_all_by_timeout(_context, host, time): def fixed_ip_get_by_address(context, address, session=None): if not session: session = get_session() - result = session.query(models.FixedIp - ).filter_by(address=address - ).filter_by(deleted=can_read_deleted(context) - ).options(joinedload('network') - ).options(joinedload('instance') - ).first() + result = session.query(models.FixedIp).\ + filter_by(address=address).\ + filter_by(deleted=can_read_deleted(context)).\ + options(joinedload('network')).\ + options(joinedload('instance')).\ + first() if not result: raise exception.NotFound('No floating ip for address %s' % address) @@ -552,10 +554,10 @@ def instance_create(context, values): def instance_data_get_for_project(context, project_id): session = get_session() result = session.query(func.count(models.Instance.id), - func.sum(models.Instance.vcpus) - ).filter_by(project_id=project_id - ).filter_by(deleted=False - ).first() + func.sum(models.Instance.vcpus)).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + first() # NOTE(vish): convert None to 0 return (result[0] or 0, result[1] or 0) @@ -575,18 +577,18 @@ def instance_get(context, instance_id, session=None): result = None if is_admin_context(context): - result = session.query(models.Instance - ).options(joinedload('security_groups') - ).filter_by(id=instance_id - ).filter_by(deleted=can_read_deleted(context) - ).first() + result = session.query(models.Instance).\ + options(joinedload('security_groups')).\ + filter_by(id=instance_id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() elif is_user_context(context): - result = session.query(models.Instance - ).options(joinedload('security_groups') - ).filter_by(project_id=context.project_id - ).filter_by(id=instance_id - ).filter_by(deleted=False - ).first() + result = session.query(models.Instance).\ + options(joinedload('security_groups')).\ + filter_by(project_id=context.project_id).\ + filter_by(id=instance_id).\ + filter_by(deleted=False).\ + first() if not result: raise exception.NotFound('No instance for id %s' % instance_id) @@ -596,22 +598,22 @@ def instance_get(context, instance_id, session=None): @require_admin_context def instance_get_all(context): session = get_session() - return session.query(models.Instance - ).options(joinedload_all('fixed_ip.floating_ips') - ).options(joinedload('security_groups') - ).filter_by(deleted=can_read_deleted(context) - ).all() + return session.query(models.Instance).\ + options(joinedload_all('fixed_ip.floating_ips')).\ + options(joinedload('security_groups')).\ + filter_by(deleted=can_read_deleted(context)).\ + all() @require_admin_context def instance_get_all_by_user(context, user_id): session = get_session() - return session.query(models.Instance - ).options(joinedload_all('fixed_ip.floating_ips') - ).options(joinedload('security_groups') - ).filter_by(deleted=can_read_deleted(context) - ).filter_by(user_id=user_id - ).all() + return session.query(models.Instance).\ + options(joinedload_all('fixed_ip.floating_ips')).\ + options(joinedload('security_groups')).\ + filter_by(deleted=can_read_deleted(context)).\ + filter_by(user_id=user_id).\ + all() @require_context @@ -619,12 +621,12 @@ def instance_get_all_by_project(context, project_id): authorize_project_context(context, project_id) session = get_session() - return session.query(models.Instance - ).options(joinedload_all('fixed_ip.floating_ips') - ).options(joinedload('security_groups') - ).filter_by(project_id=project_id - ).filter_by(deleted=can_read_deleted(context) - ).all() + return session.query(models.Instance).\ + options(joinedload_all('fixed_ip.floating_ips')).\ + options(joinedload('security_groups')).\ + filter_by(project_id=project_id).\ + filter_by(deleted=can_read_deleted(context)).\ + all() @require_context @@ -632,20 +634,20 @@ def instance_get_all_by_reservation(context, reservation_id): session = get_session() if is_admin_context(context): - return session.query(models.Instance - ).options(joinedload_all('fixed_ip.floating_ips') - ).options(joinedload('security_groups') - ).filter_by(reservation_id=reservation_id - ).filter_by(deleted=can_read_deleted(context) - ).all() + return session.query(models.Instance).\ + options(joinedload_all('fixed_ip.floating_ips')).\ + options(joinedload('security_groups')).\ + filter_by(reservation_id=reservation_id).\ + filter_by(deleted=can_read_deleted(context)).\ + all() elif is_user_context(context): - return session.query(models.Instance - ).options(joinedload_all('fixed_ip.floating_ips') - ).options(joinedload('security_groups') - ).filter_by(project_id=context.project_id - ).filter_by(reservation_id=reservation_id - ).filter_by(deleted=False - ).all() + return session.query(models.Instance).\ + options(joinedload_all('fixed_ip.floating_ips')).\ + options(joinedload('security_groups')).\ + filter_by(project_id=context.project_id).\ + filter_by(reservation_id=reservation_id).\ + filter_by(deleted=False).\ + all() @require_context @@ -653,18 +655,18 @@ def instance_get_by_internal_id(context, internal_id): session = get_session() if is_admin_context(context): - result = session.query(models.Instance - ).options(joinedload('security_groups') - ).filter_by(internal_id=internal_id - ).filter_by(deleted=can_read_deleted(context) - ).first() + result = session.query(models.Instance).\ + options(joinedload('security_groups')).\ + filter_by(internal_id=internal_id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() elif is_user_context(context): - result = session.query(models.Instance - ).options(joinedload('security_groups') - ).filter_by(project_id=context.project_id - ).filter_by(internal_id=internal_id - ).filter_by(deleted=False - ).first() + result = session.query(models.Instance).\ + options(joinedload('security_groups')).\ + filter_by(project_id=context.project_id).\ + filter_by(internal_id=internal_id).\ + filter_by(deleted=False).\ + first() if not result: raise exception.NotFound('Instance %s not found' % (internal_id)) @@ -675,9 +677,9 @@ def instance_get_by_internal_id(context, internal_id): def instance_internal_id_exists(context, internal_id, session=None): if not session: session = get_session() - return session.query( - exists().where(models.Instance.internal_id==internal_id) - ).one()[0] + return session.query(exists().\ + where(models.Instance.internal_id == internal_id)).\ + one()[0] @require_context @@ -782,11 +784,11 @@ def key_pair_get(context, user_id, name, session=None): if not session: session = get_session() - result = session.query(models.KeyPair - ).filter_by(user_id=user_id - ).filter_by(name=name - ).filter_by(deleted=can_read_deleted(context) - ).first() + result = session.query(models.KeyPair).\ + filter_by(user_id=user_id).\ + filter_by(name=name).\ + filter_by(deleted=can_read_deleted(context)).\ + first() if not result: raise exception.NotFound('no keypair for user %s, name %s' % (user_id, name)) @@ -797,10 +799,10 @@ def key_pair_get(context, user_id, name, session=None): def key_pair_get_all_by_user(context, user_id): authorize_user_context(context, user_id) session = get_session() - return session.query(models.KeyPair - ).filter_by(user_id=user_id - ).filter_by(deleted=False - ).all() + return session.query(models.KeyPair).\ + filter_by(user_id=user_id).\ + filter_by(deleted=False).\ + all() ################### @@ -810,11 +812,11 @@ def key_pair_get_all_by_user(context, user_id): def network_associate(context, project_id): session = get_session() with session.begin(): - network_ref = session.query(models.Network - ).filter_by(deleted=False - ).filter_by(project_id=None - ).with_lockmode('update' - ).first() + network_ref = session.query(models.Network).\ + filter_by(deleted=False).\ + filter_by(project_id=None).\ + with_lockmode('update').\ + first() # NOTE(vish): if with_lockmode isn't supported, as in sqlite, # then this has concurrency issues if not network_ref: @@ -827,40 +829,40 @@ def network_associate(context, project_id): @require_admin_context def network_count(context): session = get_session() - return session.query(models.Network - ).filter_by(deleted=can_read_deleted(context) - ).count() + return session.query(models.Network).\ + filter_by(deleted=can_read_deleted(context)).\ + count() @require_admin_context def network_count_allocated_ips(context, network_id): session = get_session() - return session.query(models.FixedIp - ).filter_by(network_id=network_id - ).filter_by(allocated=True - ).filter_by(deleted=False - ).count() + return session.query(models.FixedIp).\ + filter_by(network_id=network_id).\ + filter_by(allocated=True).\ + filter_by(deleted=False).\ + count() @require_admin_context def network_count_available_ips(context, network_id): session = get_session() - return session.query(models.FixedIp - ).filter_by(network_id=network_id - ).filter_by(allocated=False - ).filter_by(reserved=False - ).filter_by(deleted=False - ).count() + return session.query(models.FixedIp).\ + filter_by(network_id=network_id).\ + filter_by(allocated=False).\ + filter_by(reserved=False).\ + filter_by(deleted=False).\ + count() @require_admin_context def network_count_reserved_ips(context, network_id): session = get_session() - return session.query(models.FixedIp - ).filter_by(network_id=network_id - ).filter_by(reserved=True - ).filter_by(deleted=False - ).count() + return session.query(models.FixedIp).\ + filter_by(network_id=network_id).\ + filter_by(reserved=True).\ + filter_by(deleted=False).\ + count() @require_admin_context @@ -893,16 +895,16 @@ def network_get(context, network_id, session=None): result = None if is_admin_context(context): - result = session.query(models.Network - ).filter_by(id=network_id - ).filter_by(deleted=can_read_deleted(context) - ).first() + result = session.query(models.Network).\ + filter_by(id=network_id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() elif is_user_context(context): - result = session.query(models.Network - ).filter_by(project_id=context.project_id - ).filter_by(id=network_id - ).filter_by(deleted=False - ).first() + result = session.query(models.Network).\ + filter_by(project_id=context.project_id).\ + filter_by(id=network_id).\ + filter_by(deleted=False).\ + first() if not result: raise exception.NotFound('No network for id %s' % network_id) @@ -915,21 +917,21 @@ def network_get(context, network_id, session=None): @require_admin_context def network_get_associated_fixed_ips(context, network_id): session = get_session() - return session.query(models.FixedIp - ).options(joinedload_all('instance') - ).filter_by(network_id=network_id - ).filter(models.FixedIp.instance_id != None - ).filter_by(deleted=False - ).all() + return session.query(models.FixedIp).\ + options(joinedload_all('instance')).\ + filter_by(network_id=network_id).\ + filter(models.FixedIp.instance_id != None).\ + filter_by(deleted=False).\ + all() @require_admin_context def network_get_by_bridge(context, bridge): session = get_session() - result = session.query(models.Network - ).filter_by(bridge=bridge - ).filter_by(deleted=False - ).first() + result = session.query(models.Network).\ + filter_by(bridge=bridge).\ + filter_by(deleted=False).\ + first() if not result: raise exception.NotFound('No network for bridge %s' % bridge) @@ -939,12 +941,12 @@ def network_get_by_bridge(context, bridge): @require_admin_context def network_get_by_instance(_context, instance_id): session = get_session() - rv = session.query(models.Network - ).filter_by(deleted=False - ).join(models.Network.fixed_ips - ).filter_by(instance_id=instance_id - ).filter_by(deleted=False - ).first() + rv = session.query(models.Network).\ + filter_by(deleted=False).\ + join(models.Network.fixed_ips).\ + filter_by(instance_id=instance_id).\ + filter_by(deleted=False).\ + first() if not rv: raise exception.NotFound('No network for instance %s' % instance_id) return rv @@ -954,11 +956,11 @@ def network_get_by_instance(_context, instance_id): def network_set_host(context, network_id, host_id): session = get_session() with session.begin(): - network_ref = session.query(models.Network - ).filter_by(id=network_id - ).filter_by(deleted=False - ).with_lockmode('update' - ).first() + network_ref = session.query(models.Network).\ + filter_by(id=network_id).\ + filter_by(deleted=False).\ + with_lockmode('update').\ + first() if not network_ref: raise exception.NotFound('No network for id %s' % network_id) @@ -987,10 +989,10 @@ def network_update(context, network_id, values): @require_context def project_get_network(context, project_id): session = get_session() - rv = session.query(models.Network - ).filter_by(project_id=project_id - ).filter_by(deleted=False - ).first() + rv = session.query(models.Network).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + first() if not rv: try: return network_associate(context, project_id) @@ -998,10 +1000,10 @@ def project_get_network(context, project_id): # NOTE(vish): We hit this if there is a race and two # processes are attempting to allocate the # network at the same time - rv = session.query(models.Network - ).filter_by(project_id=project_id - ).filter_by(deleted=False - ).first() + rv = session.query(models.Network).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + first() return rv @@ -1019,9 +1021,9 @@ def queue_get_for(_context, topic, physical_node_id): @require_admin_context def export_device_count(context): session = get_session() - return session.query(models.ExportDevice - ).filter_by(deleted=can_read_deleted(context) - ).count() + return session.query(models.ExportDevice).\ + filter_by(deleted=can_read_deleted(context)).\ + count() @require_admin_context @@ -1043,18 +1045,20 @@ def auth_destroy_token(_context, token): session = get_session() session.delete(token) + def auth_get_token(_context, token_hash): session = get_session() - tk = session.query(models.AuthToken - ).filter_by(token_hash=token_hash - ).first() + tk = session.query(models.AuthToken).\ + filter_by(token_hash=token_hash).\ + first() if not tk: raise exception.NotFound('Token %s does not exist' % token_hash) return tk + def auth_create_token(_context, token): tk = models.AuthToken() - for k,v in token.iteritems(): + for k, v in token.iteritems(): tk[k] = v tk.save() return tk @@ -1068,10 +1072,10 @@ def quota_get(context, project_id, session=None): if not session: session = get_session() - result = session.query(models.Quota - ).filter_by(project_id=project_id - ).filter_by(deleted=can_read_deleted(context) - ).first() + result = session.query(models.Quota).\ + filter_by(project_id=project_id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() if not result: raise exception.NotFound('No quota for project_id %s' % project_id) @@ -1112,11 +1116,11 @@ def quota_destroy(context, project_id): def volume_allocate_shelf_and_blade(context, volume_id): session = get_session() with session.begin(): - export_device = session.query(models.ExportDevice - ).filter_by(volume=None - ).filter_by(deleted=False - ).with_lockmode('update' - ).first() + export_device = session.query(models.ExportDevice).\ + filter_by(volume=None).\ + filter_by(deleted=False).\ + with_lockmode('update').\ + first() # NOTE(vish): if with_lockmode isn't supported, as in sqlite, # then this has concurrency issues if not export_device: @@ -1134,7 +1138,8 @@ def volume_attached(context, volume_id, instance_id, mountpoint): volume_ref['status'] = 'in-use' volume_ref['mountpoint'] = mountpoint volume_ref['attach_status'] = 'attached' - volume_ref.instance = instance_get(context, instance_id, session=session) + volume_ref.instance = instance_get(context, instance_id, + session=session) volume_ref.save(session=session) @@ -1158,10 +1163,10 @@ def volume_create(context, values): def volume_data_get_for_project(context, project_id): session = get_session() result = session.query(func.count(models.Volume.id), - func.sum(models.Volume.size) - ).filter_by(project_id=project_id - ).filter_by(deleted=False - ).first() + func.sum(models.Volume.size)).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + first() # NOTE(vish): convert None to 0 return (result[0] or 0, result[1] or 0) @@ -1197,16 +1202,16 @@ def volume_get(context, volume_id, session=None): result = None if is_admin_context(context): - result = session.query(models.Volume - ).filter_by(id=volume_id - ).filter_by(deleted=can_read_deleted(context) - ).first() + result = session.query(models.Volume).\ + filter_by(id=volume_id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() elif is_user_context(context): - result = session.query(models.Volume - ).filter_by(project_id=context.project_id - ).filter_by(id=volume_id - ).filter_by(deleted=False - ).first() + result = session.query(models.Volume).\ + filter_by(project_id=context.project_id).\ + filter_by(id=volume_id).\ + filter_by(deleted=False).\ + first() if not result: raise exception.NotFound('No volume for id %s' % volume_id) @@ -1216,19 +1221,20 @@ def volume_get(context, volume_id, session=None): @require_admin_context def volume_get_all(context): session = get_session() - return session.query(models.Volume - ).filter_by(deleted=can_read_deleted(context) - ).all() + return session.query(models.Volume).\ + filter_by(deleted=can_read_deleted(context)).\ + all() + @require_context def volume_get_all_by_project(context, project_id): authorize_project_context(context, project_id) session = get_session() - return session.query(models.Volume - ).filter_by(project_id=project_id - ).filter_by(deleted=can_read_deleted(context) - ).all() + return session.query(models.Volume).\ + filter_by(project_id=project_id).\ + filter_by(deleted=can_read_deleted(context)).\ + all() @require_context @@ -1237,16 +1243,16 @@ def volume_get_by_ec2_id(context, ec2_id): result = None if is_admin_context(context): - result = session.query(models.Volume - ).filter_by(ec2_id=ec2_id - ).filter_by(deleted=can_read_deleted(context) - ).first() + result = session.query(models.Volume).\ + filter_by(ec2_id=ec2_id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() elif is_user_context(context): - result = session.query(models.Volume - ).filter_by(project_id=context.project_id - ).filter_by(ec2_id=ec2_id - ).filter_by(deleted=False - ).first() + result = session.query(models.Volume).\ + filter_by(project_id=context.project_id).\ + filter_by(ec2_id=ec2_id).\ + filter_by(deleted=False).\ + first() else: raise exception.NotAuthorized() @@ -1261,19 +1267,19 @@ def volume_ec2_id_exists(context, ec2_id, session=None): if not session: session = get_session() - return session.query(exists( - ).where(models.Volume.id==ec2_id) - ).one()[0] + return session.query(exists().\ + where(models.Volume.id == ec2_id)).\ + one()[0] @require_admin_context def volume_get_instance(context, volume_id): session = get_session() - result = session.query(models.Volume - ).filter_by(id=volume_id - ).filter_by(deleted=can_read_deleted(context) - ).options(joinedload('instance') - ).first() + result = session.query(models.Volume).\ + filter_by(id=volume_id).\ + filter_by(deleted=can_read_deleted(context)).\ + options(joinedload('instance')).\ + first() if not result: raise exception.NotFound('Volume %s not found' % ec2_id) @@ -1283,9 +1289,9 @@ def volume_get_instance(context, volume_id): @require_admin_context def volume_get_shelf_and_blade(context, volume_id): session = get_session() - result = session.query(models.ExportDevice - ).filter_by(volume_id=volume_id - ).first() + result = session.query(models.ExportDevice).\ + filter_by(volume_id=volume_id).\ + first() if not result: raise exception.NotFound('No export device found for volume %s' % volume_id) @@ -1309,10 +1315,10 @@ def volume_update(context, volume_id, values): @require_context def security_group_get_all(context): session = get_session() - return session.query(models.SecurityGroup - ).filter_by(deleted=can_read_deleted(context) - ).options(joinedload_all('rules') - ).all() + return session.query(models.SecurityGroup).\ + filter_by(deleted=can_read_deleted(context)).\ + options(joinedload_all('rules')).\ + all() @require_context @@ -1320,18 +1326,18 @@ def security_group_get(context, security_group_id, session=None): if not session: session = get_session() if is_admin_context(context): - result = session.query(models.SecurityGroup - ).filter_by(deleted=can_read_deleted(context), - ).filter_by(id=security_group_id - ).options(joinedload_all('rules') - ).first() + result = session.query(models.SecurityGroup).\ + filter_by(deleted=can_read_deleted(context),).\ + filter_by(id=security_group_id).\ + options(joinedload_all('rules')).\ + first() else: - result = session.query(models.SecurityGroup - ).filter_by(deleted=False - ).filter_by(id=security_group_id - ).filter_by(project_id=context.project_id - ).options(joinedload_all('rules') - ).first() + result = session.query(models.SecurityGroup).\ + filter_by(deleted=False).\ + filter_by(id=security_group_id).\ + filter_by(project_id=context.project_id).\ + options(joinedload_all('rules')).\ + first() if not result: raise exception.NotFound("No secuity group with id %s" % security_group_id) @@ -1341,13 +1347,13 @@ def security_group_get(context, security_group_id, session=None): @require_context def security_group_get_by_name(context, project_id, group_name): session = get_session() - result = session.query(models.SecurityGroup - ).filter_by(project_id=project_id - ).filter_by(name=group_name - ).filter_by(deleted=False - ).options(joinedload_all('rules') - ).options(joinedload_all('instances') - ).first() + result = session.query(models.SecurityGroup).\ + filter_by(project_id=project_id).\ + filter_by(name=group_name).\ + filter_by(deleted=False).\ + options(joinedload_all('rules')).\ + options(joinedload_all('instances')).\ + first() if not result: raise exception.NotFound( 'No security group named %s for project: %s' \ @@ -1358,23 +1364,23 @@ def security_group_get_by_name(context, project_id, group_name): @require_context def security_group_get_by_project(context, project_id): session = get_session() - return session.query(models.SecurityGroup - ).filter_by(project_id=project_id - ).filter_by(deleted=False - ).options(joinedload_all('rules') - ).all() + return session.query(models.SecurityGroup).\ + filter_by(project_id=project_id).\ + filter_by(deleted=False).\ + options(joinedload_all('rules')).\ + all() @require_context def security_group_get_by_instance(context, instance_id): session = get_session() - return session.query(models.SecurityGroup - ).filter_by(deleted=False - ).options(joinedload_all('rules') - ).join(models.SecurityGroup.instances - ).filter_by(id=instance_id - ).filter_by(deleted=False - ).all() + return session.query(models.SecurityGroup).\ + filter_by(deleted=False).\ + options(joinedload_all('rules')).\ + join(models.SecurityGroup.instances).\ + filter_by(id=instance_id).\ + filter_by(deleted=False).\ + all() @require_context @@ -1409,6 +1415,7 @@ def security_group_destroy(context, security_group_id): 'where group_id=:id', {'id': security_group_id}) + @require_context def security_group_destroy_all(context, session=None): if not session: @@ -1427,16 +1434,16 @@ def security_group_rule_get(context, security_group_rule_id, session=None): if not session: session = get_session() if is_admin_context(context): - result = session.query(models.SecurityGroupIngressRule - ).filter_by(deleted=can_read_deleted(context) - ).filter_by(id=security_group_rule_id - ).first() + result = session.query(models.SecurityGroupIngressRule).\ + filter_by(deleted=can_read_deleted(context)).\ + filter_by(id=security_group_rule_id).\ + first() else: # TODO(vish): Join to group and check for project_id - result = session.query(models.SecurityGroupIngressRule - ).filter_by(deleted=False - ).filter_by(id=security_group_rule_id - ).first() + result = session.query(models.SecurityGroupIngressRule).\ + filter_by(deleted=False).\ + filter_by(id=security_group_rule_id).\ + first() if not result: raise exception.NotFound("No secuity group rule with id %s" % security_group_rule_id) @@ -1451,6 +1458,7 @@ def security_group_rule_create(context, values): security_group_rule_ref.save() return security_group_rule_ref + @require_context def security_group_rule_destroy(context, security_group_rule_id): session = get_session() @@ -1468,10 +1476,10 @@ def user_get(context, id, session=None): if not session: session = get_session() - result = session.query(models.User - ).filter_by(id=id - ).filter_by(deleted=can_read_deleted(context) - ).first() + result = session.query(models.User).\ + filter_by(id=id).\ + filter_by(deleted=can_read_deleted(context)).\ + first() if not result: raise exception.NotFound('No user for id %s' % id) @@ -1484,10 +1492,10 @@ def user_get_by_access_key(context, access_key, session=None): if not session: session = get_session() - result = session.query(models.User - ).filter_by(access_key=access_key - ).filter_by(deleted=can_read_deleted(context) - ).first() + result = session.query(models.User).\ + filter_by(access_key=access_key).\ + filter_by(deleted=can_read_deleted(context)).\ + first() if not result: raise exception.NotFound('No user for access key %s' % access_key) @@ -1508,21 +1516,21 @@ def user_create(_context, values): def user_delete(context, id): session = get_session() with session.begin(): - session.execute('delete from user_project_association where user_id=:id', - {'id': id}) - session.execute('delete from user_role_association where user_id=:id', - {'id': id}) - session.execute('delete from user_project_role_association where user_id=:id', - {'id': id}) + session.execute('delete from user_project_association ' + 'where user_id=:id', {'id': id}) + session.execute('delete from user_role_association ' + 'where user_id=:id', {'id': id}) + session.execute('delete from user_project_role_association ' + 'where user_id=:id', {'id': id}) user_ref = user_get(context, id, session=session) session.delete(user_ref) def user_get_all(context): session = get_session() - return session.query(models.User - ).filter_by(deleted=can_read_deleted(context) - ).all() + return session.query(models.User).\ + filter_by(deleted=can_read_deleted(context)).\ + all() def project_create(_context, values): @@ -1547,11 +1555,11 @@ def project_get(context, id, session=None): if not session: session = get_session() - result = session.query(models.Project - ).filter_by(deleted=False - ).filter_by(id=id - ).options(joinedload_all('members') - ).first() + result = session.query(models.Project).\ + filter_by(deleted=False).\ + filter_by(id=id).\ + options(joinedload_all('members')).\ + first() if not result: raise exception.NotFound("No project with id %s" % id) @@ -1561,18 +1569,18 @@ def project_get(context, id, session=None): def project_get_all(context): session = get_session() - return session.query(models.Project - ).filter_by(deleted=can_read_deleted(context) - ).options(joinedload_all('members') - ).all() + return session.query(models.Project).\ + filter_by(deleted=can_read_deleted(context)).\ + options(joinedload_all('members')).\ + all() def project_get_by_user(context, user_id): session = get_session() - user = session.query(models.User - ).filter_by(deleted=can_read_deleted(context) - ).options(joinedload_all('projects') - ).first() + user = session.query(models.User).\ + filter_by(deleted=can_read_deleted(context)).\ + options(joinedload_all('projects')).\ + first() return user.projects @@ -1607,10 +1615,10 @@ def project_update(context, project_id, values): def project_delete(context, id): session = get_session() with session.begin(): - session.execute('delete from user_project_association where project_id=:id', - {'id': id}) - session.execute('delete from user_project_role_association where project_id=:id', - {'id': id}) + session.execute('delete from user_project_association ' + 'where project_id=:id', {'id': id}) + session.execute('delete from user_project_role_association ' + 'where project_id=:id', {'id': id}) project_ref = project_get(context, id, session=session) session.delete(project_ref) @@ -1625,29 +1633,30 @@ def user_get_roles(context, user_id): def user_get_roles_for_project(context, user_id, project_id): session = get_session() with session.begin(): - res = session.query(models.UserProjectRoleAssociation - ).filter_by(user_id=user_id - ).filter_by(project_id=project_id - ).all() + res = session.query(models.UserProjectRoleAssociation).\ + filter_by(user_id=user_id).\ + filter_by(project_id=project_id).\ + all() return [association.role for association in res] + def user_remove_project_role(context, user_id, project_id, role): session = get_session() with session.begin(): - session.execute('delete from user_project_role_association where ' + \ - 'user_id=:user_id and project_id=:project_id and ' + \ - 'role=:role', { 'user_id' : user_id, - 'project_id' : project_id, - 'role' : role }) + session.execute('delete from user_project_role_association where ' + 'user_id=:user_id and project_id=:project_id and ' + 'role=:role', {'user_id': user_id, + 'project_id': project_id, + 'role': role}) def user_remove_role(context, user_id, role): session = get_session() with session.begin(): - res = session.query(models.UserRoleAssociation - ).filter_by(user_id=user_id - ).filter_by(role=role - ).all() + res = session.query(models.UserRoleAssociation).\ + filter_by(user_id=user_id).\ + filter_by(role=role).\ + all() for role in res: session.delete(role) @@ -1656,7 +1665,8 @@ def user_add_role(context, user_id, role): session = get_session() with session.begin(): user_ref = user_get(context, user_id, session=session) - models.UserRoleAssociation(user=user_ref, role=role).save(session=session) + models.UserRoleAssociation(user=user_ref, role=role).\ + save(session=session) def user_add_project_role(context, user_id, project_id, role): @@ -1672,12 +1682,11 @@ def user_add_project_role(context, user_id, project_id, role): ################### - @require_admin_context def host_get_networks(context, host): session = get_session() with session.begin(): - return session.query(models.Network - ).filter_by(deleted=False - ).filter_by(host=host - ).all() + return session.query(models.Network).\ + filter_by(deleted=False).\ + filter_by(host=host).\ + all() diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index a63bca2b0..38c96bdec 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -134,8 +134,8 @@ class NovaBase(object): # """Represents a host where services are running""" # __tablename__ = 'hosts' # id = Column(String(255), primary_key=True) -# -# + + class Service(BASE, NovaBase): """Represents a running service on a host""" __tablename__ = 'services' @@ -277,7 +277,8 @@ class Quota(BASE, NovaBase): class ExportDevice(BASE, NovaBase): """Represates a shelf and blade that a volume can be exported on""" __tablename__ = 'export_devices' - __table_args__ = (schema.UniqueConstraint("shelf_id", "blade_id"), {'mysql_engine': 'InnoDB'}) + __table_args__ = (schema.UniqueConstraint("shelf_id", "blade_id"), + {'mysql_engine': 'InnoDB'}) id = Column(Integer, primary_key=True) shelf_id = Column(Integer) blade_id = Column(Integer) @@ -308,10 +309,13 @@ class SecurityGroup(BASE, NovaBase): instances = relationship(Instance, secondary="security_group_instance_association", - primaryjoin="and_(SecurityGroup.id == SecurityGroupInstanceAssociation.security_group_id," - "SecurityGroup.deleted == False)", - secondaryjoin="and_(SecurityGroupInstanceAssociation.instance_id == Instance.id," - "Instance.deleted == False)", + primaryjoin='and_(' + 'SecurityGroup.id == ' + 'SecurityGroupInstanceAssociation.security_group_id,' + 'SecurityGroup.deleted == False)', + secondaryjoin='and_(' + 'SecurityGroupInstanceAssociation.instance_id == Instance.id,' + 'Instance.deleted == False)', backref='security_groups') @property @@ -330,11 +334,12 @@ class SecurityGroupIngressRule(BASE, NovaBase): parent_group_id = Column(Integer, ForeignKey('security_groups.id')) parent_group = relationship("SecurityGroup", backref="rules", - foreign_keys=parent_group_id, - primaryjoin="and_(SecurityGroupIngressRule.parent_group_id == SecurityGroup.id," - "SecurityGroupIngressRule.deleted == False)") + foreign_keys=parent_group_id, + primaryjoin='and_(' + 'SecurityGroupIngressRule.parent_group_id == SecurityGroup.id,' + 'SecurityGroupIngressRule.deleted == False)') - protocol = Column(String(5)) # "tcp", "udp", or "icmp" + protocol = Column(String(5)) # "tcp", "udp", or "icmp" from_port = Column(Integer) to_port = Column(Integer) cidr = Column(String(255)) @@ -414,8 +419,9 @@ class FixedIp(BASE, NovaBase): instance = relationship(Instance, backref=backref('fixed_ip', uselist=False), foreign_keys=instance_id, - primaryjoin='and_(FixedIp.instance_id==Instance.id,' - 'FixedIp.deleted==False)') + primaryjoin='and_(' + 'FixedIp.instance_id == Instance.id,' + 'FixedIp.deleted == False)') allocated = Column(Boolean, default=False) leased = Column(Boolean, default=False) reserved = Column(Boolean, default=False) @@ -455,13 +461,13 @@ class UserProjectRoleAssociation(BASE, NovaBase): __tablename__ = 'user_project_role_association' user_id = Column(String(255), primary_key=True) user = relationship(User, - primaryjoin=user_id==User.id, + primaryjoin=user_id == User.id, foreign_keys=[User.id], uselist=False) project_id = Column(String(255), primary_key=True) project = relationship(Project, - primaryjoin=project_id==Project.id, + primaryjoin=project_id == Project.id, foreign_keys=[Project.id], uselist=False) @@ -485,7 +491,6 @@ class UserProjectAssociation(BASE, NovaBase): project_id = Column(String(255), ForeignKey(Project.id), primary_key=True) - class FloatingIp(BASE, NovaBase): """Represents a floating ip that dynamically forwards to a fixed ip""" __tablename__ = 'floating_ips' @@ -495,8 +500,9 @@ class FloatingIp(BASE, NovaBase): fixed_ip = relationship(FixedIp, backref=backref('floating_ips'), foreign_keys=fixed_ip_id, - primaryjoin='and_(FloatingIp.fixed_ip_id==FixedIp.id,' - 'FloatingIp.deleted==False)') + primaryjoin='and_(' + 'FloatingIp.fixed_ip_id == FixedIp.id,' + 'FloatingIp.deleted == False)') project_id = Column(String(255)) host = Column(String(255)) # , ForeignKey('hosts.id')) @@ -507,7 +513,7 @@ def register_models(): models = (Service, Instance, Volume, ExportDevice, FixedIp, FloatingIp, Network, SecurityGroup, SecurityGroupIngressRule, SecurityGroupInstanceAssociation, - AuthToken, User, Project) # , Image, Host + AuthToken, User, Project) # , Image, Host engine = create_engine(FLAGS.sql_connection, echo=False) for model in models: model.metadata.create_all(engine) diff --git a/nova/db/sqlalchemy/session.py b/nova/db/sqlalchemy/session.py index 826754f6a..e0d84c107 100644 --- a/nova/db/sqlalchemy/session.py +++ b/nova/db/sqlalchemy/session.py @@ -29,6 +29,7 @@ FLAGS = flags.FLAGS _ENGINE = None _MAKER = None + def get_session(autocommit=True, expire_on_commit=False): """Helper method to grab session""" global _ENGINE @@ -39,5 +40,5 @@ def get_session(autocommit=True, expire_on_commit=False): _MAKER = (sessionmaker(bind=_ENGINE, autocommit=autocommit, expire_on_commit=expire_on_commit)) - session = _MAKER() + session = _MAKER() return session -- cgit From 8c6ef1380426aca6d7c82e3489789ccde8ee047c Mon Sep 17 00:00:00 2001 From: Eric Day Date: Thu, 21 Oct 2010 17:15:21 -0700 Subject: PEP8 cleanup in nova/*, except for tests. There should be no functional changes here, just style changes to get violations down. --- nova/auth/dbdriver.py | 51 ++++++++------- nova/auth/fakeldap.py | 11 ++-- nova/auth/ldapdriver.py | 12 ++-- nova/auth/manager.py | 2 +- nova/cloudpipe/pipelib.py | 30 +++++---- nova/compute/disk.py | 15 +++-- nova/compute/monitor.py | 111 +++++++++++++++------------------ nova/compute/power_state.py | 13 ++-- nova/image/service.py | 17 ++--- nova/image/services/glance/__init__.py | 8 +-- nova/network/linux_net.py | 10 ++- nova/network/manager.py | 5 +- nova/objectstore/bucket.py | 20 +++--- nova/objectstore/handler.py | 21 ++++--- nova/objectstore/image.py | 41 ++++++------ nova/objectstore/stored.py | 4 +- nova/scheduler/driver.py | 2 + nova/scheduler/manager.py | 3 +- nova/scheduler/simple.py | 1 + nova/virt/fake.py | 1 + nova/virt/images.py | 4 +- nova/virt/libvirt_conn.py | 97 ++++++++++++++++------------ nova/virt/xenapi.py | 28 +++++---- nova/volume/driver.py | 1 - 24 files changed, 273 insertions(+), 235 deletions(-) diff --git a/nova/auth/dbdriver.py b/nova/auth/dbdriver.py index 648d6e828..a1584322b 100644 --- a/nova/auth/dbdriver.py +++ b/nova/auth/dbdriver.py @@ -47,19 +47,23 @@ class DbDriver(object): def get_user(self, uid): """Retrieve user by id""" - return self._db_user_to_auth_user(db.user_get(context.get_admin_context(), uid)) + user = db.user_get(context.get_admin_context(), uid) + return self._db_user_to_auth_user(user) def get_user_from_access_key(self, access): """Retrieve user by access key""" - return self._db_user_to_auth_user(db.user_get_by_access_key(context.get_admin_context(), access)) + user = db.user_get_by_access_key(context.get_admin_context(), access) + return self._db_user_to_auth_user(user) def get_project(self, pid): """Retrieve project by id""" - return self._db_project_to_auth_projectuser(db.project_get(context.get_admin_context(), pid)) + project = db.project_get(context.get_admin_context(), pid) + return self._db_project_to_auth_projectuser(project) def get_users(self): """Retrieve list of users""" - return [self._db_user_to_auth_user(user) for user in db.user_get_all(context.get_admin_context())] + return [self._db_user_to_auth_user(user) + for user in db.user_get_all(context.get_admin_context())] def get_projects(self, uid=None): """Retrieve list of projects""" @@ -71,11 +75,10 @@ class DbDriver(object): def create_user(self, name, access_key, secret_key, is_admin): """Create a user""" - values = { 'id' : name, - 'access_key' : access_key, - 'secret_key' : secret_key, - 'is_admin' : is_admin - } + values = {'id': name, + 'access_key': access_key, + 'secret_key': secret_key, + 'is_admin': is_admin} try: user_ref = db.user_create(context.get_admin_context(), values) return self._db_user_to_auth_user(user_ref) @@ -83,18 +86,19 @@ class DbDriver(object): raise exception.Duplicate('User %s already exists' % name) def _db_user_to_auth_user(self, user_ref): - return { 'id' : user_ref['id'], - 'name' : user_ref['id'], - 'access' : user_ref['access_key'], - 'secret' : user_ref['secret_key'], - 'admin' : user_ref['is_admin'] } + return {'id': user_ref['id'], + 'name': user_ref['id'], + 'access': user_ref['access_key'], + 'secret': user_ref['secret_key'], + 'admin': user_ref['is_admin']} def _db_project_to_auth_projectuser(self, project_ref): - return { 'id' : project_ref['id'], - 'name' : project_ref['name'], - 'project_manager_id' : project_ref['project_manager'], - 'description' : project_ref['description'], - 'member_ids' : [member['id'] for member in project_ref['members']] } + member_ids = [member['id'] for member in project_ref['members']] + return {'id': project_ref['id'], + 'name': project_ref['name'], + 'project_manager_id': project_ref['project_manager'], + 'description': project_ref['description'], + 'member_ids': member_ids} def create_project(self, name, manager_uid, description=None, member_uids=None): @@ -121,10 +125,10 @@ class DbDriver(object): % member_uid) members.add(member) - values = { 'id' : name, - 'name' : name, - 'project_manager' : manager['id'], - 'description': description } + values = {'id': name, + 'name': name, + 'project_manager': manager['id'], + 'description': description} try: project = db.project_create(context.get_admin_context(), values) @@ -244,4 +248,3 @@ class DbDriver(object): if not project: raise exception.NotFound('Project "%s" not found' % project_id) return user, project - diff --git a/nova/auth/fakeldap.py b/nova/auth/fakeldap.py index 3e92c38f6..cf3a84a5d 100644 --- a/nova/auth/fakeldap.py +++ b/nova/auth/fakeldap.py @@ -35,6 +35,7 @@ flags.DEFINE_integer('redis_port', 6379, 'Port that redis is running on.') flags.DEFINE_integer('redis_db', 0, 'Multiple DB keeps tests away') + class Redis(object): def __init__(self): if hasattr(self.__class__, '_instance'): @@ -51,19 +52,19 @@ class Redis(object): SCOPE_BASE = 0 -SCOPE_ONELEVEL = 1 # not implemented +SCOPE_ONELEVEL = 1 # Not implemented SCOPE_SUBTREE = 2 MOD_ADD = 0 MOD_DELETE = 1 MOD_REPLACE = 2 -class NO_SUCH_OBJECT(Exception): # pylint: disable-msg=C0103 +class NO_SUCH_OBJECT(Exception): # pylint: disable-msg=C0103 """Duplicate exception class from real LDAP module.""" pass -class OBJECT_CLASS_VIOLATION(Exception): # pylint: disable-msg=C0103 +class OBJECT_CLASS_VIOLATION(Exception): # pylint: disable-msg=C0103 """Duplicate exception class from real LDAP module.""" pass @@ -251,8 +252,6 @@ class FakeLDAP(object): return objects @property - def __redis_prefix(self): # pylint: disable-msg=R0201 + def __redis_prefix(self): # pylint: disable-msg=R0201 """Get the prefix to use for all redis keys.""" return 'ldap:' - - diff --git a/nova/auth/ldapdriver.py b/nova/auth/ldapdriver.py index 640ea169e..ceade1d65 100644 --- a/nova/auth/ldapdriver.py +++ b/nova/auth/ldapdriver.py @@ -294,24 +294,26 @@ class LdapDriver(object): def __find_dns(self, dn, query=None, scope=None): """Find dns by query""" - if scope is None: # one of the flags is 0!! + if scope is None: + # One of the flags is 0! scope = self.ldap.SCOPE_SUBTREE try: res = self.conn.search_s(dn, scope, query) except self.ldap.NO_SUCH_OBJECT: return [] - # just return the DNs + # Just return the DNs return [dn for dn, _attributes in res] def __find_objects(self, dn, query=None, scope=None): """Find objects by query""" - if scope is None: # one of the flags is 0!! + if scope is None: + # One of the flags is 0! scope = self.ldap.SCOPE_SUBTREE try: res = self.conn.search_s(dn, scope, query) except self.ldap.NO_SUCH_OBJECT: return [] - # just return the attributes + # Just return the attributes return [attributes for dn, attributes in res] def __find_role_dns(self, tree): @@ -480,6 +482,6 @@ class LdapDriver(object): class FakeLdapDriver(LdapDriver): """Fake Ldap Auth driver""" - def __init__(self): # pylint: disable-msg=W0231 + def __init__(self): # pylint: disable-msg=W0231 __import__('nova.auth.fakeldap') self.ldap = sys.modules['nova.auth.fakeldap'] diff --git a/nova/auth/manager.py b/nova/auth/manager.py index bf7ca8a95..001a96875 100644 --- a/nova/auth/manager.py +++ b/nova/auth/manager.py @@ -23,7 +23,7 @@ Nova authentication management import logging import os import shutil -import string # pylint: disable-msg=W0402 +import string # pylint: disable-msg=W0402 import tempfile import uuid import zipfile diff --git a/nova/cloudpipe/pipelib.py b/nova/cloudpipe/pipelib.py index 4fc2c85cb..3472201cd 100644 --- a/nova/cloudpipe/pipelib.py +++ b/nova/cloudpipe/pipelib.py @@ -49,7 +49,7 @@ class CloudPipe(object): self.manager = manager.AuthManager() def launch_vpn_instance(self, project_id): - logging.debug( "Launching VPN for %s" % (project_id)) + logging.debug("Launching VPN for %s" % (project_id)) project = self.manager.get_project(project_id) # Make a payload.zip tmpfolder = tempfile.mkdtemp() @@ -57,16 +57,18 @@ class CloudPipe(object): zippath = os.path.join(tmpfolder, filename) z = zipfile.ZipFile(zippath, "w", zipfile.ZIP_DEFLATED) - z.write(FLAGS.boot_script_template,'autorun.sh') + z.write(FLAGS.boot_script_template, 'autorun.sh') z.close() key_name = self.setup_key_pair(project.project_manager_id, project_id) zippy = open(zippath, "r") - context = context.RequestContext(user=project.project_manager, project=project) + context = context.RequestContext(user=project.project_manager, + project=project) reservation = self.controller.run_instances(context, - # run instances expects encoded userdata, it is decoded in the get_metadata_call - # autorun.sh also decodes the zip file, hence the double encoding + # Run instances expects encoded userdata, it is decoded in the + # get_metadata_call. autorun.sh also decodes the zip file, hence + # the double encoding. user_data=zippy.read().encode("base64").encode("base64"), max_count=1, min_count=1, @@ -79,12 +81,14 @@ class CloudPipe(object): def setup_key_pair(self, user_id, project_id): key_name = '%s%s' % (project_id, FLAGS.vpn_key_suffix) try: - private_key, fingerprint = self.manager.generate_key_pair(user_id, key_name) + private_key, fingerprint = self.manager.generate_key_pair(user_id, + key_name) try: key_dir = os.path.join(FLAGS.keys_path, user_id) if not os.path.exists(key_dir): os.makedirs(key_dir) - with open(os.path.join(key_dir, '%s.pem' % key_name),'w') as f: + file_name = os.path.join(key_dir, '%s.pem' % key_name) + with open(file_name, 'w') as f: f.write(private_key) except: pass @@ -95,9 +99,13 @@ class CloudPipe(object): # def setup_secgroups(self, username): # conn = self.euca.connection_for(username) # try: - # secgroup = conn.create_security_group("vpn-secgroup", "vpn-secgroup") - # secgroup.authorize(ip_protocol = "udp", from_port = "1194", to_port = "1194", cidr_ip = "0.0.0.0/0") - # secgroup.authorize(ip_protocol = "tcp", from_port = "80", to_port = "80", cidr_ip = "0.0.0.0/0") - # secgroup.authorize(ip_protocol = "tcp", from_port = "22", to_port = "22", cidr_ip = "0.0.0.0/0") + # secgroup = conn.create_security_group("vpn-secgroup", + # "vpn-secgroup") + # secgroup.authorize(ip_protocol = "udp", from_port = "1194", + # to_port = "1194", cidr_ip = "0.0.0.0/0") + # secgroup.authorize(ip_protocol = "tcp", from_port = "80", + # to_port = "80", cidr_ip = "0.0.0.0/0") + # secgroup.authorize(ip_protocol = "tcp", from_port = "22", + # to_port = "22", cidr_ip = "0.0.0.0/0") # except: # pass diff --git a/nova/compute/disk.py b/nova/compute/disk.py index 447a29004..2b2340b97 100644 --- a/nova/compute/disk.py +++ b/nova/compute/disk.py @@ -72,12 +72,12 @@ def partition(infile, outfile, local_bytes=0, resize=True, " by sector size: %d / %d", local_bytes, sector_size) local_sectors = local_bytes / sector_size - mbr_last = 62 # a - primary_first = mbr_last + 1 # b - primary_last = primary_first + primary_sectors - 1 # c - local_first = primary_last + 1 # d - local_last = local_first + local_sectors - 1 # e - last_sector = local_last # e + mbr_last = 62 # a + primary_first = mbr_last + 1 # b + primary_last = primary_first + primary_sectors - 1 # c + local_first = primary_last + 1 # d + local_last = local_first + local_sectors - 1 # e + last_sector = local_last # e # create an empty file yield execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' @@ -157,7 +157,7 @@ def inject_data(image, key=None, net=None, partition=None, execute=None): @defer.inlineCallbacks def _inject_key_into_fs(key, fs, execute=None): sshdir = os.path.join(os.path.join(fs, 'root'), '.ssh') - yield execute('sudo mkdir -p %s' % sshdir) # existing dir doesn't matter + yield execute('sudo mkdir -p %s' % sshdir) # existing dir doesn't matter yield execute('sudo chown root %s' % sshdir) yield execute('sudo chmod 700 %s' % sshdir) keyfile = os.path.join(sshdir, 'authorized_keys') @@ -169,4 +169,3 @@ def _inject_net_into_fs(net, fs, execute=None): netfile = os.path.join(os.path.join(os.path.join( fs, 'etc'), 'network'), 'interfaces') yield execute('sudo tee %s' % netfile, net) - diff --git a/nova/compute/monitor.py b/nova/compute/monitor.py index 268864900..d0154600f 100644 --- a/nova/compute/monitor.py +++ b/nova/compute/monitor.py @@ -85,8 +85,7 @@ RRD_VALUES = { 'RRA:MAX:0.5:6:800', 'RRA:MAX:0.5:24:800', 'RRA:MAX:0.5:444:800', - ] - } + ]} utcnow = datetime.datetime.utcnow @@ -97,15 +96,12 @@ def update_rrd(instance, name, data): Updates the specified RRD file. """ filename = os.path.join(instance.get_rrd_path(), '%s.rrd' % name) - + if not os.path.exists(filename): init_rrd(instance, name) - + timestamp = int(time.mktime(utcnow().timetuple())) - rrdtool.update ( - filename, - '%d:%s' % (timestamp, data) - ) + rrdtool.update(filename, '%d:%s' % (timestamp, data)) def init_rrd(instance, name): @@ -113,29 +109,28 @@ def init_rrd(instance, name): Initializes the specified RRD file. """ path = os.path.join(FLAGS.monitoring_rrd_path, instance.instance_id) - + if not os.path.exists(path): os.makedirs(path) - + filename = os.path.join(path, '%s.rrd' % name) - + if not os.path.exists(filename): - rrdtool.create ( + rrdtool.create( filename, '--step', '%d' % FLAGS.monitoring_instances_step, '--start', '0', - *RRD_VALUES[name] - ) + *RRD_VALUES[name]) + - def graph_cpu(instance, duration): """ Creates a graph of cpu usage for the specified instance and duration. """ path = instance.get_rrd_path() filename = os.path.join(path, 'cpu-%s.png' % duration) - - rrdtool.graph ( + + rrdtool.graph( filename, '--disable-rrdtool-tag', '--imgformat', 'PNG', @@ -146,9 +141,8 @@ def graph_cpu(instance, duration): '-l', '0', '-u', '100', 'DEF:cpu=%s:cpu:AVERAGE' % os.path.join(path, 'cpu.rrd'), - 'AREA:cpu#eacc00:% CPU', - ) - + 'AREA:cpu#eacc00:% CPU',) + store_graph(instance.instance_id, filename) @@ -158,8 +152,8 @@ def graph_net(instance, duration): """ path = instance.get_rrd_path() filename = os.path.join(path, 'net-%s.png' % duration) - - rrdtool.graph ( + + rrdtool.graph( filename, '--disable-rrdtool-tag', '--imgformat', 'PNG', @@ -174,20 +168,19 @@ def graph_net(instance, duration): 'DEF:rx=%s:rx:AVERAGE' % os.path.join(path, 'net.rrd'), 'DEF:tx=%s:tx:AVERAGE' % os.path.join(path, 'net.rrd'), 'AREA:rx#00FF00:In traffic', - 'LINE1:tx#0000FF:Out traffic', - ) - + 'LINE1:tx#0000FF:Out traffic',) + store_graph(instance.instance_id, filename) - + def graph_disk(instance, duration): """ Creates a graph of disk usage for the specified duration. - """ + """ path = instance.get_rrd_path() filename = os.path.join(path, 'disk-%s.png' % duration) - - rrdtool.graph ( + + rrdtool.graph( filename, '--disable-rrdtool-tag', '--imgformat', 'PNG', @@ -202,9 +195,8 @@ def graph_disk(instance, duration): 'DEF:rd=%s:rd:AVERAGE' % os.path.join(path, 'disk.rrd'), 'DEF:wr=%s:wr:AVERAGE' % os.path.join(path, 'disk.rrd'), 'AREA:rd#00FF00:Read', - 'LINE1:wr#0000FF:Write', - ) - + 'LINE1:wr#0000FF:Write',) + store_graph(instance.instance_id, filename) @@ -224,17 +216,16 @@ def store_graph(instance_id, filename): is_secure=False, calling_format=boto.s3.connection.OrdinaryCallingFormat(), port=FLAGS.s3_port, - host=FLAGS.s3_host - ) + host=FLAGS.s3_host) bucket_name = '_%s.monitor' % instance_id - + # Object store isn't creating the bucket like it should currently # when it is first requested, so have to catch and create manually. try: bucket = s3.get_bucket(bucket_name) except Exception: bucket = s3.create_bucket(bucket_name) - + key = boto.s3.Key(bucket) key.key = os.path.basename(filename) key.set_contents_from_filename(filename) @@ -247,18 +238,18 @@ class Instance(object): self.last_updated = datetime.datetime.min self.cputime = 0 self.cputime_last_updated = None - + init_rrd(self, 'cpu') init_rrd(self, 'net') init_rrd(self, 'disk') - + def needs_update(self): """ Indicates whether this instance is due to have its statistics updated. """ delta = utcnow() - self.last_updated return delta.seconds >= FLAGS.monitoring_instances_step - + def update(self): """ Updates the instances statistics and stores the resulting graphs @@ -271,7 +262,7 @@ class Instance(object): if data != None: logging.debug('CPU: %s', data) update_rrd(self, 'cpu', data) - + data = self.fetch_net_stats() logging.debug('NET: %s', data) update_rrd(self, 'net', data) @@ -279,7 +270,7 @@ class Instance(object): data = self.fetch_disk_stats() logging.debug('DISK: %s', data) update_rrd(self, 'disk', data) - + # TODO(devcamcar): Turn these into pool.ProcessPool.execute() calls # and make the methods @defer.inlineCallbacks. graph_cpu(self, '1d') @@ -297,13 +288,13 @@ class Instance(object): logging.exception('unexpected error during update') self.last_updated = utcnow() - + def get_rrd_path(self): """ Returns the path to where RRD files are stored. """ return os.path.join(FLAGS.monitoring_rrd_path, self.instance_id) - + def fetch_cpu_stats(self): """ Returns cpu usage statistics for this instance. @@ -327,17 +318,17 @@ class Instance(object): # Calculate the number of seconds between samples. d = self.cputime_last_updated - cputime_last_updated t = d.days * 86400 + d.seconds - + logging.debug('t = %d', t) # Calculate change over time in number of nanoseconds of CPU time used. cputime_delta = self.cputime - cputime_last - + logging.debug('cputime_delta = %s', cputime_delta) # Get the number of virtual cpus in this domain. vcpus = int(info['num_cpu']) - + logging.debug('vcpus = %d', vcpus) # Calculate CPU % used and cap at 100. @@ -349,9 +340,9 @@ class Instance(object): """ rd = 0 wr = 0 - + disks = self.conn.get_disks(self.instance_id) - + # Aggregate the read and write totals. for disk in disks: try: @@ -363,7 +354,7 @@ class Instance(object): logging.error('Cannot get blockstats for "%s" on "%s"', disk, self.instance_id) raise - + return '%d:%d' % (rd, wr) def fetch_net_stats(self): @@ -372,9 +363,9 @@ class Instance(object): """ rx = 0 tx = 0 - + interfaces = self.conn.get_interfaces(self.instance_id) - + # Aggregate the in and out totals. for interface in interfaces: try: @@ -385,7 +376,7 @@ class Instance(object): logging.error('Cannot get ifstats for "%s" on "%s"', interface, self.instance_id) raise - + return '%d:%d' % (rx, tx) @@ -400,16 +391,16 @@ class InstanceMonitor(object, service.Service): """ self._instances = {} self._loop = task.LoopingCall(self.updateInstances) - + def startService(self): self._instances = {} self._loop.start(interval=FLAGS.monitoring_instances_delay) service.Service.startService(self) - + def stopService(self): self._loop.stop() service.Service.stopService(self) - + def updateInstances(self): """ Update resource usage for all running instances. @@ -420,20 +411,20 @@ class InstanceMonitor(object, service.Service): logging.exception('unexpected exception getting connection') time.sleep(FLAGS.monitoring_instances_delay) return - + domain_ids = conn.list_instances() try: - self.updateInstances_(conn, domain_ids) + self.updateInstances_(conn, domain_ids) except Exception, exn: - logging.exception('updateInstances_') + logging.exception('updateInstances_') def updateInstances_(self, conn, domain_ids): for domain_id in domain_ids: - if not domain_id in self._instances: + if not domain_id in self._instances: instance = Instance(conn, domain_id) self._instances[domain_id] = instance logging.debug('Found instance: %s', domain_id) - + for key in self._instances.keys(): instance = self._instances[key] if instance.needs_update(): diff --git a/nova/compute/power_state.py b/nova/compute/power_state.py index b27aa4677..cefdf2d9e 100644 --- a/nova/compute/power_state.py +++ b/nova/compute/power_state.py @@ -30,12 +30,11 @@ CRASHED = 0x06 def name(code): d = { - NOSTATE : 'pending', - RUNNING : 'running', - BLOCKED : 'blocked', - PAUSED : 'paused', + NOSTATE: 'pending', + RUNNING: 'running', + BLOCKED: 'blocked', + PAUSED: 'paused', SHUTDOWN: 'shutdown', - SHUTOFF : 'shutdown', - CRASHED : 'crashed', - } + SHUTOFF: 'shutdown', + CRASHED: 'crashed'} return d[code] diff --git a/nova/image/service.py b/nova/image/service.py index ebdc65fef..37cadddcc 100644 --- a/nova/image/service.py +++ b/nova/image/service.py @@ -30,7 +30,8 @@ flags.DEFINE_string('glance_teller_address', 'http://127.0.0.1', flags.DEFINE_string('glance_teller_port', '9191', 'Port for Glance\'s Teller service') flags.DEFINE_string('glance_parallax_address', 'http://127.0.0.1', - 'IP address or URL where Glance\'s Parallax service resides') + 'IP address or URL where Glance\'s Parallax service ' + 'resides') flags.DEFINE_string('glance_parallax_port', '9292', 'Port for Glance\'s Parallax service') @@ -120,10 +121,10 @@ class BaseImageService(object): def delete(self, image_id): """ - Delete the given image. - + Delete the given image. + :raises NotFound if the image does not exist. - + """ raise NotImplementedError @@ -131,14 +132,14 @@ class BaseImageService(object): class LocalImageService(BaseImageService): """Image service storing images to local disk. - + It assumes that image_ids are integers.""" def __init__(self): self._path = "/tmp/nova/images" try: os.makedirs(self._path) - except OSError: # exists + except OSError: # Exists pass def _path_to(self, image_id): @@ -156,7 +157,7 @@ class LocalImageService(BaseImageService): def show(self, id): try: - return pickle.load(open(self._path_to(id))) + return pickle.load(open(self._path_to(id))) except IOError: raise exception.NotFound @@ -164,7 +165,7 @@ class LocalImageService(BaseImageService): """ Store the image data and return the new image id. """ - id = random.randint(0, 2**32-1) + id = random.randint(0, 2 ** 32 - 1) data['id'] = id self.update(id, data) return id diff --git a/nova/image/services/glance/__init__.py b/nova/image/services/glance/__init__.py index 6df6f2dcf..f1d05f0bc 100644 --- a/nova/image/services/glance/__init__.py +++ b/nova/image/services/glance/__init__.py @@ -30,6 +30,7 @@ import nova.image.service FLAGS = flags.FLAGS + class TellerClient(object): def __init__(self): @@ -153,7 +154,6 @@ class ParallaxClient(object): class GlanceImageService(nova.image.service.BaseImageService): - """Provides storage and retrieval of disk image objects within Glance.""" def __init__(self): @@ -202,10 +202,10 @@ class GlanceImageService(nova.image.service.BaseImageService): def delete(self, image_id): """ - Delete the given image. - + Delete the given image. + :raises NotFound if the image does not exist. - + """ self.parallax.delete_image_metadata(image_id) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index f75a079d9..7b323efa1 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -53,6 +53,7 @@ flags.DEFINE_bool('use_nova_chains', False, DEFAULT_PORTS = [("tcp", 80), ("tcp", 22), ("udp", 1194), ("tcp", 443)] + def init_host(): """Basic networking setup goes here""" # NOTE(devcamcar): Cloud public DNAT entries, CloudPipe port @@ -72,6 +73,7 @@ def init_host(): _confirm_rule("POSTROUTING", "-t nat -s %(range)s -d %(range)s -j ACCEPT" % {'range': FLAGS.fixed_range}) + def bind_floating_ip(floating_ip): """Bind ip to public interface""" _execute("sudo ip addr add %s dev %s" % (floating_ip, @@ -103,7 +105,7 @@ def ensure_floating_forward(floating_ip, fixed_ip): _confirm_rule("FORWARD", "-d %s -p icmp -j ACCEPT" % (fixed_ip)) for (protocol, port) in DEFAULT_PORTS: - _confirm_rule("FORWARD","-d %s -p %s --dport %s -j ACCEPT" + _confirm_rule("FORWARD", "-d %s -p %s --dport %s -j ACCEPT" % (fixed_ip, protocol, port)) @@ -189,7 +191,8 @@ def update_dhcp(context, network_id): # if dnsmasq is already running, then tell it to reload if pid: - out, _err = _execute('cat /proc/%d/cmdline' % pid, check_exit_code=False) + out, _err = _execute('cat /proc/%d/cmdline' % pid, + check_exit_code=False) if conffile in out: try: _execute('sudo kill -HUP %d' % pid) @@ -233,7 +236,8 @@ def _confirm_rule(chain, cmd): """Delete and re-add iptables rule""" if FLAGS.use_nova_chains: chain = "nova_%s" % chain.lower() - _execute("sudo iptables --delete %s %s" % (chain, cmd), check_exit_code=False) + _execute("sudo iptables --delete %s %s" % (chain, cmd), + check_exit_code=False) _execute("sudo iptables -I %s %s" % (chain, cmd)) diff --git a/nova/network/manager.py b/nova/network/manager.py index c7080ccd8..fddb77663 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -49,7 +49,8 @@ flags.DEFINE_string('vpn_ip', utils.get_my_ip(), flags.DEFINE_integer('vpn_start', 1000, 'First Vpn port for private networks') flags.DEFINE_integer('network_size', 256, 'Number of addresses in each private subnet') -flags.DEFINE_string('floating_range', '4.4.4.0/24', 'Floating IP address block') +flags.DEFINE_string('floating_range', '4.4.4.0/24', + 'Floating IP address block') flags.DEFINE_string('fixed_range', '10.0.0.0/8', 'Fixed IP address block') flags.DEFINE_integer('cnt_vpn_clients', 5, 'Number of addresses reserved for vpn clients') @@ -287,7 +288,6 @@ class FlatManager(NetworkManager): self.db.network_update(context, network_id, net) - class FlatDHCPManager(NetworkManager): """Flat networking with dhcp""" @@ -432,4 +432,3 @@ class VlanManager(NetworkManager): """Number of reserved ips at the top of the range""" parent_reserved = super(VlanManager, self)._top_reserved_ips return parent_reserved + FLAGS.cnt_vpn_clients - diff --git a/nova/objectstore/bucket.py b/nova/objectstore/bucket.py index cfe5b14d8..0ba4934d1 100644 --- a/nova/objectstore/bucket.py +++ b/nova/objectstore/bucket.py @@ -69,7 +69,8 @@ class Bucket(object): """Create a new bucket owned by a project. @bucket_name: a string representing the name of the bucket to create - @context: a nova.auth.api.ApiContext object representing who owns the bucket. + @context: a nova.auth.api.ApiContext object representing who owns the + bucket. Raises: NotAuthorized: if the bucket is already exists or has invalid name @@ -77,12 +78,12 @@ class Bucket(object): path = os.path.abspath(os.path.join( FLAGS.buckets_path, bucket_name)) if not path.startswith(os.path.abspath(FLAGS.buckets_path)) or \ - os.path.exists(path): - raise exception.NotAuthorized() + os.path.exists(path): + raise exception.NotAuthorized() os.makedirs(path) - with open(path+'.json', 'w') as f: + with open(path + '.json', 'w') as f: json.dump({'ownerId': context.project_id}, f) @property @@ -99,22 +100,25 @@ class Bucket(object): @property def owner_id(self): try: - with open(self.path+'.json') as f: + with open(self.path + '.json') as f: return json.load(f)['ownerId'] except: return None def is_authorized(self, context): try: - return context.user.is_admin() or self.owner_id == context.project_id + return context.user.is_admin() or \ + self.owner_id == context.project_id except Exception, e: return False def list_keys(self, prefix='', marker=None, max_keys=1000, terse=False): object_names = [] + path_length = len(self.path) for root, dirs, files in os.walk(self.path): for file_name in files: - object_names.append(os.path.join(root, file_name)[len(self.path)+1:]) + object_name = os.path.join(root, file_name)[path_length + 1:] + object_names.append(object_name) object_names.sort() contents = [] @@ -164,7 +168,7 @@ class Bucket(object): if len(os.listdir(self.path)) > 0: raise exception.NotEmpty() os.rmdir(self.path) - os.remove(self.path+'.json') + os.remove(self.path + '.json') def __getitem__(self, key): return stored.Object(self, key) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index cc0d8ffc9..074eea601 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -136,6 +136,7 @@ def get_context(request): logging.debug("Authentication Failure: %s", ex) raise exception.NotAuthorized() + class ErrorHandlingResource(resource.Resource): """Maps exceptions to 404 / 401 codes. Won't work for exceptions thrown after NOT_DONE_YET is returned. @@ -162,7 +163,7 @@ class S3(ErrorHandlingResource): def __init__(self): ErrorHandlingResource.__init__(self) - def getChild(self, name, request): # pylint: disable-msg=C0103 + def getChild(self, name, request): # pylint: disable-msg=C0103 """Returns either the image or bucket resource""" request.context = get_context(request) if name == '': @@ -172,7 +173,7 @@ class S3(ErrorHandlingResource): else: return BucketResource(name) - def render_GET(self, request): # pylint: disable-msg=R0201 + def render_GET(self, request): # pylint: disable-msg=R0201 """Renders the GET request for a list of buckets as XML""" logging.debug('List of buckets requested') buckets = [b for b in bucket.Bucket.all() \ @@ -321,11 +322,13 @@ class ImageResource(ErrorHandlingResource): if not self.img.is_authorized(request.context, True): raise exception.NotAuthorized() return static.File(self.img.image_path, - defaultType='application/octet-stream' - ).render_GET(request) + defaultType='application/octet-stream').\ + render_GET(request) + class ImagesResource(resource.Resource): """A web resource representing a list of images""" + def getChild(self, name, _request): """Returns itself or an ImageResource if no name given""" if name == '': @@ -333,7 +336,7 @@ class ImagesResource(resource.Resource): else: return ImageResource(name) - def render_GET(self, request): # pylint: disable-msg=R0201 + def render_GET(self, request): # pylint: disable-msg=R0201 """ returns a json listing of all images that a user has permissions to see """ @@ -362,7 +365,7 @@ class ImagesResource(resource.Resource): request.finish() return server.NOT_DONE_YET - def render_PUT(self, request): # pylint: disable-msg=R0201 + def render_PUT(self, request): # pylint: disable-msg=R0201 """ create a new registered image """ image_id = get_argument(request, 'image_id', u'') @@ -383,7 +386,7 @@ class ImagesResource(resource.Resource): p.start() return '' - def render_POST(self, request): # pylint: disable-msg=R0201 + def render_POST(self, request): # pylint: disable-msg=R0201 """Update image attributes: public/private""" # image_id required for all requests @@ -397,7 +400,7 @@ class ImagesResource(resource.Resource): if operation: # operation implies publicity toggle logging.debug("handling publicity toggle") - image_object.set_public(operation=='add') + image_object.set_public(operation == 'add') else: # other attributes imply update logging.debug("update user fields") @@ -407,7 +410,7 @@ class ImagesResource(resource.Resource): image_object.update_user_editable_fields(clean_args) return '' - def render_DELETE(self, request): # pylint: disable-msg=R0201 + def render_DELETE(self, request): # pylint: disable-msg=R0201 """Delete a registered image""" image_id = get_argument(request, "image_id", u"") image_object = image.Image(image_id) diff --git a/nova/objectstore/image.py b/nova/objectstore/image.py index 413b269b7..b7b2ec6ab 100644 --- a/nova/objectstore/image.py +++ b/nova/objectstore/image.py @@ -48,8 +48,8 @@ class Image(object): self.image_id = image_id self.path = os.path.abspath(os.path.join(FLAGS.images_path, image_id)) if not self.path.startswith(os.path.abspath(FLAGS.images_path)) or \ - not os.path.isdir(self.path): - raise exception.NotFound + not os.path.isdir(self.path): + raise exception.NotFound @property def image_path(self): @@ -127,8 +127,8 @@ class Image(object): a string of the image id for the kernel @type ramdisk: bool or str - @param ramdisk: either TRUE meaning this partition is a ramdisk image or - a string of the image id for the ramdisk + @param ramdisk: either TRUE meaning this partition is a ramdisk image + or a string of the image id for the ramdisk @type public: bool @@ -160,8 +160,7 @@ class Image(object): 'isPublic': public, 'architecture': 'x86_64', 'imageType': image_type, - 'state': 'available' - } + 'state': 'available'} if type(kernel) is str and len(kernel) > 0: info['kernelId'] = kernel @@ -180,7 +179,7 @@ class Image(object): os.makedirs(image_path) bucket_name = image_location.split("/")[0] - manifest_path = image_location[len(bucket_name)+1:] + manifest_path = image_location[len(bucket_name) + 1:] bucket_object = bucket.Bucket(bucket_name) manifest = ElementTree.fromstring(bucket_object[manifest_path].read()) @@ -204,10 +203,9 @@ class Image(object): 'imageId': image_id, 'imageLocation': image_location, 'imageOwnerId': context.project_id, - 'isPublic': False, # FIXME: grab public from manifest - 'architecture': 'x86_64', # FIXME: grab architecture from manifest - 'imageType' : image_type - } + 'isPublic': False, # FIXME: grab public from manifest + 'architecture': 'x86_64', # FIXME: grab architecture from manifest + 'imageType': image_type} if kernel_id: info['kernelId'] = kernel_id @@ -230,24 +228,29 @@ class Image(object): write_state('decrypting') # FIXME: grab kernelId and ramdiskId from bundle manifest - encrypted_key = binascii.a2b_hex(manifest.find("image/ec2_encrypted_key").text) - encrypted_iv = binascii.a2b_hex(manifest.find("image/ec2_encrypted_iv").text) + hex_key = manifest.find("image/ec2_encrypted_key").text + encrypted_key = binascii.a2b_hex(hex_key) + hex_iv = manifest.find("image/ec2_encrypted_iv").text + encrypted_iv = binascii.a2b_hex(hex_iv) cloud_private_key = os.path.join(FLAGS.ca_path, "private/cakey.pem") decrypted_filename = os.path.join(image_path, 'image.tar.gz') - Image.decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, cloud_private_key, decrypted_filename) + Image.decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, + cloud_private_key, decrypted_filename) write_state('untarring') image_file = Image.untarzip_image(image_path, decrypted_filename) - shutil.move(os.path.join(image_path, image_file), os.path.join(image_path, 'image')) + shutil.move(os.path.join(image_path, image_file), + os.path.join(image_path, 'image')) write_state('available') os.unlink(decrypted_filename) os.unlink(encrypted_filename) @staticmethod - def decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, cloud_private_key, decrypted_filename): + def decrypt_image(encrypted_filename, encrypted_key, encrypted_iv, + cloud_private_key, decrypted_filename): key, err = utils.execute( 'openssl rsautl -decrypt -inkey %s' % cloud_private_key, process_input=encrypted_key, @@ -259,13 +262,15 @@ class Image(object): process_input=encrypted_iv, check_exit_code=False) if err: - raise exception.Error("Failed to decrypt initialization vector: %s" % err) + raise exception.Error("Failed to decrypt initialization " + "vector: %s" % err) _out, err = utils.execute( 'openssl enc -d -aes-128-cbc -in %s -K %s -iv %s -out %s' % (encrypted_filename, key, iv, decrypted_filename), check_exit_code=False) if err: - raise exception.Error("Failed to decrypt image file %s : %s" % (encrypted_filename, err)) + raise exception.Error("Failed to decrypt image file %s : %s" % + (encrypted_filename, err)) @staticmethod def untarzip_image(path, filename): diff --git a/nova/objectstore/stored.py b/nova/objectstore/stored.py index 9829194cb..a3f6e9c0b 100644 --- a/nova/objectstore/stored.py +++ b/nova/objectstore/stored.py @@ -50,8 +50,8 @@ class Object(object): return os.path.getmtime(self.path) def read(self): - """ read all contents of key into memory and return """ - return self.file.read() + """ read all contents of key into memory and return """ + return self.file.read() @property def file(self): diff --git a/nova/scheduler/driver.py b/nova/scheduler/driver.py index c89d25a47..f271d573f 100644 --- a/nova/scheduler/driver.py +++ b/nova/scheduler/driver.py @@ -31,10 +31,12 @@ FLAGS = flags.FLAGS flags.DEFINE_integer('service_down_time', 60, 'maximum time since last checkin for up service') + class NoValidHost(exception.Error): """There is no valid host for the command.""" pass + class Scheduler(object): """The base class that all Scheduler clases should inherit from.""" diff --git a/nova/scheduler/manager.py b/nova/scheduler/manager.py index b3b2b4dce..60a3d2b4b 100644 --- a/nova/scheduler/manager.py +++ b/nova/scheduler/manager.py @@ -56,7 +56,8 @@ class SchedulerManager(manager.Manager): driver_method = 'schedule_%s' % method elevated = context.elevated() try: - host = getattr(self.driver, driver_method)(elevated, *args, **kwargs) + host = getattr(self.driver, driver_method)(elevated, *args, + **kwargs) except AttributeError: host = self.driver.schedule(elevated, topic, *args, **kwargs) diff --git a/nova/scheduler/simple.py b/nova/scheduler/simple.py index fdaff74d8..7f5093656 100644 --- a/nova/scheduler/simple.py +++ b/nova/scheduler/simple.py @@ -36,6 +36,7 @@ flags.DEFINE_integer("max_gigabytes", 10000, flags.DEFINE_integer("max_networks", 1000, "maximum number of networks to allow per host") + class SimpleScheduler(chance.ChanceScheduler): """Implements Naive Scheduler that tries to find least loaded host.""" diff --git a/nova/virt/fake.py b/nova/virt/fake.py index dc6112f20..eaa2261f5 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -226,6 +226,7 @@ class FakeConnection(object): def get_console_output(self, instance): return 'FAKE CONSOLE OUTPUT' + class FakeInstance(object): def __init__(self): self._state = power_state.NOSTATE diff --git a/nova/virt/images.py b/nova/virt/images.py index dc50764d9..981aa5cf3 100644 --- a/nova/virt/images.py +++ b/nova/virt/images.py @@ -62,8 +62,8 @@ def _fetch_s3_image(image, path, user, project): headers['Authorization'] = 'AWS %s:%s' % (access, signature) cmd = ['/usr/bin/curl', '--fail', '--silent', url] - for (k,v) in headers.iteritems(): - cmd += ['-H', '%s: %s' % (k,v)] + for (k, v) in headers.iteritems(): + cmd += ['-H', '%s: %s' % (k, v)] cmd += ['-o', path] return process.SharedPool().execute(executable=cmd[0], args=cmd[1:]) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 7250d31e5..509ed97a0 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -62,7 +62,8 @@ flags.DEFINE_string('injected_network_template', 'Template file for injected network') flags.DEFINE_string('libvirt_type', 'kvm', - 'Libvirt domain type (valid options are: kvm, qemu, uml, xen)') + 'Libvirt domain type (valid options are: ' + 'kvm, qemu, uml, xen)') flags.DEFINE_string('libvirt_uri', '', 'Override the default libvirt URI (which is dependent' @@ -96,7 +97,8 @@ class LibvirtConnection(object): def _conn(self): if not self._wrapped_conn or not self._test_connection(): logging.debug('Connecting to libvirt: %s' % self.libvirt_uri) - self._wrapped_conn = self._connect(self.libvirt_uri, self.read_only) + self._wrapped_conn = self._connect(self.libvirt_uri, + self.read_only) return self._wrapped_conn def _test_connection(self): @@ -150,6 +152,7 @@ class LibvirtConnection(object): # WE'LL save this for when we do shutdown, # instead of destroy - but destroy returns immediately timer = task.LoopingCall(f=None) + def _wait_for_shutdown(): try: state = self.get_info(instance['name'])['state'] @@ -164,6 +167,7 @@ class LibvirtConnection(object): power_state.SHUTDOWN) timer.stop() d.callback(None) + timer.f = _wait_for_shutdown timer.start(interval=0.5, now=True) return d @@ -201,6 +205,7 @@ class LibvirtConnection(object): d = defer.Deferred() timer = task.LoopingCall(f=None) + def _wait_for_reboot(): try: state = self.get_info(instance['name'])['state'] @@ -217,6 +222,7 @@ class LibvirtConnection(object): power_state.SHUTDOWN) timer.stop() d.callback(None) + timer.f = _wait_for_reboot timer.start(interval=0.5, now=True) yield d @@ -229,7 +235,8 @@ class LibvirtConnection(object): instance['id'], power_state.NOSTATE, 'launching') - yield NWFilterFirewall(self._conn).setup_nwfilters_for_instance(instance) + yield NWFilterFirewall(self._conn).\ + setup_nwfilters_for_instance(instance) yield self._create_image(instance, xml) yield self._conn.createXML(xml, 0) # TODO(termie): this should actually register @@ -238,6 +245,7 @@ class LibvirtConnection(object): local_d = defer.Deferred() timer = task.LoopingCall(f=None) + def _wait_for_boot(): try: state = self.get_info(instance['name'])['state'] @@ -265,8 +273,9 @@ class LibvirtConnection(object): if virsh_output.startswith('/dev/'): logging.info('cool, it\'s a device') - d = process.simple_execute("sudo dd if=%s iflag=nonblock" % virsh_output, check_exit_code=False) - d.addCallback(lambda r:r[0]) + d = process.simple_execute("sudo dd if=%s iflag=nonblock" % + virsh_output, check_exit_code=False) + d.addCallback(lambda r: r[0]) return d else: return '' @@ -285,11 +294,15 @@ class LibvirtConnection(object): @exception.wrap_exception def get_console_output(self, instance): - console_log = os.path.join(FLAGS.instances_path, instance['name'], 'console.log') - d = process.simple_execute('sudo chown %d %s' % (os.getuid(), console_log)) + console_log = os.path.join(FLAGS.instances_path, instance['name'], + 'console.log') + d = process.simple_execute('sudo chown %d %s' % (os.getuid(), + console_log)) if FLAGS.libvirt_type == 'xen': # Xen is spethial - d.addCallback(lambda _: process.simple_execute("virsh ttyconsole %s" % instance['name'])) + d.addCallback(lambda _: + process.simple_execute("virsh ttyconsole %s" % + instance['name'])) d.addCallback(self._flush_xen_console) d.addCallback(self._append_to_file, console_log) else: @@ -297,7 +310,6 @@ class LibvirtConnection(object): d.addCallback(self._dump_file) return d - @defer.inlineCallbacks def _create_image(self, inst, libvirt_xml): # syntactic nicety @@ -309,7 +321,6 @@ class LibvirtConnection(object): yield process.simple_execute('mkdir -p %s' % basepath()) yield process.simple_execute('chmod 0777 %s' % basepath()) - # TODO(termie): these are blocking calls, it would be great # if they weren't. logging.info('instance %s: Creating image', inst['name']) @@ -317,17 +328,21 @@ class LibvirtConnection(object): f.write(libvirt_xml) f.close() - os.close(os.open(basepath('console.log'), os.O_CREAT | os.O_WRONLY, 0660)) + os.close(os.open(basepath('console.log'), os.O_CREAT | os.O_WRONLY, + 0660)) user = manager.AuthManager().get_user(inst['user_id']) project = manager.AuthManager().get_project(inst['project_id']) if not os.path.exists(basepath('disk')): - yield images.fetch(inst.image_id, basepath('disk-raw'), user, project) + yield images.fetch(inst.image_id, basepath('disk-raw'), user, + project) if not os.path.exists(basepath('kernel')): - yield images.fetch(inst.kernel_id, basepath('kernel'), user, project) + yield images.fetch(inst.kernel_id, basepath('kernel'), user, + project) if not os.path.exists(basepath('ramdisk')): - yield images.fetch(inst.ramdisk_id, basepath('ramdisk'), user, project) + yield images.fetch(inst.ramdisk_id, basepath('ramdisk'), user, + project) execute = lambda cmd, process_input=None, check_exit_code=True: \ process.simple_execute(cmd=cmd, @@ -339,8 +354,8 @@ class LibvirtConnection(object): network_ref = db.network_get_by_instance(context.get_admin_context(), inst['id']) if network_ref['injected']: - address = db.instance_get_fixed_address(context.get_admin_context(), - inst['id']) + admin_context = context.get_admin_context() + address = db.instance_get_fixed_address(admin_context, inst['id']) with open(FLAGS.injected_network_template) as f: net = f.read() % {'address': address, 'netmask': network_ref['netmask'], @@ -354,7 +369,8 @@ class LibvirtConnection(object): if net: logging.info('instance %s: injecting net into image %s', inst['name'], inst.image_id) - yield disk.inject_data(basepath('disk-raw'), key, net, execute=execute) + yield disk.inject_data(basepath('disk-raw'), key, net, + execute=execute) if os.path.exists(basepath('disk')): yield process.simple_execute('rm -f %s' % basepath('disk')) @@ -377,7 +393,8 @@ class LibvirtConnection(object): network = db.project_get_network(context.get_admin_context(), instance['project_id']) # FIXME(vish): stick this in db - instance_type = instance_types.INSTANCE_TYPES[instance['instance_type']] + instance_type = instance['instance_type'] + instance_type = instance_types.INSTANCE_TYPES[instance_type] ip_address = db.instance_get_fixed_address(context.get_admin_context(), instance['id']) # Assume that the gateway also acts as the dhcp server. @@ -391,7 +408,7 @@ class LibvirtConnection(object): 'bridge_name': network['bridge'], 'mac_address': instance['mac_address'], 'ip_address': ip_address, - 'dhcp_server': dhcp_server } + 'dhcp_server': dhcp_server} libvirt_xml = self.libvirt_xml % xml_info logging.debug('instance %s: finished toXML method', instance['name']) @@ -506,7 +523,6 @@ class LibvirtConnection(object): domain = self._conn.lookupByName(instance_name) return domain.interfaceStats(interface) - def refresh_security_group(self, security_group_id): fw = NWFilterFirewall(self._conn) fw.ensure_security_group_filter(security_group_id) @@ -557,7 +573,6 @@ class NWFilterFirewall(object): def __init__(self, get_connection): self._conn = get_connection - nova_base_filter = ''' 26717364-50cf-42d1-8185-29bf893ab110 @@ -578,7 +593,8 @@ class NWFilterFirewall(object): srcportstart='68' dstportstart='67'/> - + @@ -588,8 +604,8 @@ class NWFilterFirewall(object): def nova_base_ipv4_filter(self): retval = "" for protocol in ['tcp', 'udp', 'icmp']: - for direction,action,priority in [('out','accept', 399), - ('inout','drop', 400)]: + for direction, action, priority in [('out', 'accept', 399), + ('inout', 'drop', 400)]: retval += """ <%s /> """ % (action, direction, @@ -597,12 +613,11 @@ class NWFilterFirewall(object): retval += '' return retval - def nova_base_ipv6_filter(self): retval = "" for protocol in ['tcp', 'udp', 'icmp']: - for direction,action,priority in [('out','accept',399), - ('inout','drop',400)]: + for direction, action, priority in [('out', 'accept', 399), + ('inout', 'drop', 400)]: retval += """ <%s-ipv6 /> """ % (action, direction, @@ -610,7 +625,6 @@ class NWFilterFirewall(object): retval += '' return retval - def nova_project_filter(self, project, net, mask): retval = "" % project for protocol in ['tcp', 'udp', 'icmp']: @@ -620,14 +634,12 @@ class NWFilterFirewall(object): retval += '' return retval - def _define_filter(self, xml): if callable(xml): xml = xml() d = threads.deferToThread(self._conn.nwfilterDefineXML, xml) return d - @staticmethod def _get_net_and_mask(cidr): net = IPy.IP(cidr) @@ -646,9 +658,9 @@ class NWFilterFirewall(object): yield self._define_filter(self.nova_dhcp_filter) yield self._define_filter(self.nova_base_filter) - nwfilter_xml = ("\n" + - " \n" - ) % instance['name'] + nwfilter_xml = "\n" \ + " \n" % \ + instance['name'] if FLAGS.allow_project_net_traffic: network_ref = db.project_get_network(context.get_admin_context(), @@ -658,14 +670,14 @@ class NWFilterFirewall(object): net, mask) yield self._define_filter(project_filter) - nwfilter_xml += (" \n" - ) % instance['project_id'] + nwfilter_xml += " \n" % \ + instance['project_id'] for security_group in instance.security_groups: yield self.ensure_security_group_filter(security_group['id']) - nwfilter_xml += (" \n" - ) % security_group['id'] + nwfilter_xml += " \n" % \ + security_group['id'] nwfilter_xml += "" yield self._define_filter(nwfilter_xml) @@ -675,7 +687,6 @@ class NWFilterFirewall(object): return self._define_filter( self.security_group_to_nwfilter_xml(security_group_id)) - def security_group_to_nwfilter_xml(self, security_group_id): security_group = db.security_group_get(context.get_admin_context(), security_group_id) @@ -684,12 +695,15 @@ class NWFilterFirewall(object): rule_xml += "" if rule.cidr: net, mask = self._get_net_and_mask(rule.cidr) - rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % (rule.protocol, net, mask) + rule_xml += "<%s srcipaddr='%s' srcipmask='%s' " % \ + (rule.protocol, net, mask) if rule.protocol in ['tcp', 'udp']: rule_xml += "dstportstart='%s' dstportend='%s' " % \ (rule.from_port, rule.to_port) elif rule.protocol == 'icmp': - logging.info('rule.protocol: %r, rule.from_port: %r, rule.to_port: %r' % (rule.protocol, rule.from_port, rule.to_port)) + logging.info('rule.protocol: %r, rule.from_port: %r, ' + 'rule.to_port: %r' % + (rule.protocol, rule.from_port, rule.to_port)) if rule.from_port != -1: rule_xml += "type='%s' " % rule.from_port if rule.to_port != -1: @@ -697,5 +711,6 @@ class NWFilterFirewall(object): rule_xml += '/>\n' rule_xml += "\n" - xml = '''%s''' % (security_group_id, rule_xml,) + xml = "%s" % \ + (security_group_id, rule_xml,) return xml diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py index 04e830b64..a17e405ab 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi.py @@ -75,12 +75,11 @@ flags.DEFINE_float('xenapi_task_poll_interval', XENAPI_POWER_STATE = { - 'Halted' : power_state.SHUTDOWN, - 'Running' : power_state.RUNNING, - 'Paused' : power_state.PAUSED, - 'Suspended': power_state.SHUTDOWN, # FIXME - 'Crashed' : power_state.CRASHED -} + 'Halted': power_state.SHUTDOWN, + 'Running': power_state.RUNNING, + 'Paused': power_state.PAUSED, + 'Suspended': power_state.SHUTDOWN, # FIXME + 'Crashed': power_state.CRASHED} def get_connection(_): @@ -90,12 +89,15 @@ def get_connection(_): # library when not using XenAPI. global XenAPI if XenAPI is None: - XenAPI = __import__('XenAPI') + XenAPI = __import__('XenAPI') url = FLAGS.xenapi_connection_url username = FLAGS.xenapi_connection_username password = FLAGS.xenapi_connection_password if not url or password is None: - raise Exception('Must specify xenapi_connection_url, xenapi_connection_username (optionally), and xenapi_connection_password to use connection_type=xenapi') + raise Exception('Must specify xenapi_connection_url, ' + 'xenapi_connection_username (optionally), and ' + 'xenapi_connection_password to use ' + 'connection_type=xenapi') return XenAPIConnection(url, username, password) @@ -141,7 +143,7 @@ class XenAPIConnection(object): def _create_vm(self, instance, kernel, ramdisk): """Create a VM record. Returns a Deferred that gives the new VM reference.""" - + instance_type = instance_types.INSTANCE_TYPES[instance.instance_type] mem = str(long(instance_type['memory_mb']) * 1024 * 1024) vcpus = str(instance_type['vcpus']) @@ -183,7 +185,7 @@ class XenAPIConnection(object): def _create_vbd(self, vm_ref, vdi_ref, userdevice, bootable): """Create a VBD record. Returns a Deferred that gives the new VBD reference.""" - + vbd_rec = {} vbd_rec['VM'] = vm_ref vbd_rec['VDI'] = vdi_ref @@ -207,10 +209,10 @@ class XenAPIConnection(object): def _create_vif(self, vm_ref, network_ref, mac_address): """Create a VIF record. Returns a Deferred that gives the new VIF reference.""" - + vif_rec = {} vif_rec['device'] = '0' - vif_rec['network']= network_ref + vif_rec['network'] = network_ref vif_rec['VM'] = vm_ref vif_rec['MAC'] = mac_address vif_rec['MTU'] = '1500' @@ -303,7 +305,7 @@ class XenAPIConnection(object): def _lookup_blocking(self, i): vms = self._conn.xenapi.VM.get_by_name_label(i) - n = len(vms) + n = len(vms) if n == 0: return None elif n > 1: diff --git a/nova/volume/driver.py b/nova/volume/driver.py index cca619550..3fa29ba37 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -61,7 +61,6 @@ class AOEDriver(object): "Try number %s", tries) yield self._execute("sleep %s" % tries ** 2) - @defer.inlineCallbacks def create_volume(self, volume_name, size): """Creates a logical volume""" -- cgit From a53aea8fedcd9cfab8b54eaaff6a1ae1d2c60684 Mon Sep 17 00:00:00 2001 From: Eric Day Date: Fri, 22 Oct 2010 00:48:27 -0700 Subject: PEP8 cleanup in nova/tests, except for tests. There should be no functional changes here, just style changes to get violations down. --- nova/tests/access_unittest.py | 7 +- nova/tests/api/__init__.py | 4 +- nova/tests/api/fakes.py | 18 +++++ nova/tests/api/openstack/__init__.py | 14 ++-- nova/tests/api/openstack/fakes.py | 26 ++++---- nova/tests/api/openstack/test_auth.py | 25 ++++++- nova/tests/api/openstack/test_faults.py | 20 +++++- nova/tests/api/openstack/test_images.py | 23 ++++--- nova/tests/api/openstack/test_ratelimiting.py | 11 ++- nova/tests/api/openstack/test_servers.py | 52 +++++++-------- nova/tests/api/test_wsgi.py | 4 +- nova/tests/api_integration.py | 8 +-- nova/tests/api_unittest.py | 24 ++++--- nova/tests/auth_unittest.py | 10 ++- nova/tests/cloud_unittest.py | 36 +++++----- nova/tests/flags_unittest.py | 5 +- nova/tests/network_unittest.py | 51 ++++++++------ nova/tests/objectstore_unittest.py | 7 +- nova/tests/process_unittest.py | 3 + nova/tests/quota_unittest.py | 7 +- nova/tests/rpc_unittest.py | 9 ++- nova/tests/scheduler_unittest.py | 2 + nova/tests/service_unittest.py | 4 +- nova/tests/validator_unittest.py | 3 +- nova/tests/virt_unittest.py | 96 +++++++++++++-------------- nova/tests/volume_unittest.py | 9 ++- 26 files changed, 290 insertions(+), 188 deletions(-) diff --git a/nova/tests/access_unittest.py b/nova/tests/access_unittest.py index 8167259c4..0f66c0a26 100644 --- a/nova/tests/access_unittest.py +++ b/nova/tests/access_unittest.py @@ -29,9 +29,12 @@ from nova.auth import manager FLAGS = flags.FLAGS + + class Context(object): pass + class AccessTestCase(test.TrialTestCase): def setUp(self): super(AccessTestCase, self).setUp() @@ -56,9 +59,11 @@ class AccessTestCase(test.TrialTestCase): self.project.add_role(self.testnet, 'netadmin') self.project.add_role(self.testsys, 'sysadmin') #user is set in each test + def noopWSGIApp(environ, start_response): start_response('200 OK', []) return [''] + self.mw = ec2.Authorizer(noopWSGIApp) self.mw.action_roles = {'str': { '_allow_all': ['all'], @@ -80,7 +85,7 @@ class AccessTestCase(test.TrialTestCase): def response_status(self, user, methodName): ctxt = context.RequestContext(user, self.project) - environ = {'ec2.context' : ctxt, + environ = {'ec2.context': ctxt, 'ec2.controller': 'some string', 'ec2.action': methodName} req = webob.Request.blank('/', environ) diff --git a/nova/tests/api/__init__.py b/nova/tests/api/__init__.py index a0681f414..46f09e906 100644 --- a/nova/tests/api/__init__.py +++ b/nova/tests/api/__init__.py @@ -66,8 +66,7 @@ class Test(unittest.TestCase): def test_metadata(self): def go(url): - result = self._request(url, 'ec2', - REMOTE_ADDR='128.192.151.2') + result = self._request(url, 'ec2', REMOTE_ADDR='128.192.151.2') # Each should get to the ORM layer and fail to find the IP self.assertRaises(nova.exception.NotFound, go, '/latest/') self.assertRaises(nova.exception.NotFound, go, '/2009-04-04/') @@ -78,6 +77,5 @@ class Test(unittest.TestCase): self.assertTrue('2007-12-15\n' in result.body) - if __name__ == '__main__': unittest.main() diff --git a/nova/tests/api/fakes.py b/nova/tests/api/fakes.py index d0a2cc027..0aedcaff0 100644 --- a/nova/tests/api/fakes.py +++ b/nova/tests/api/fakes.py @@ -1,6 +1,24 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + import webob.dec from nova import wsgi + class APIStub(object): """Class to verify request and mark it was called.""" @webob.dec.wsgify diff --git a/nova/tests/api/openstack/__init__.py b/nova/tests/api/openstack/__init__.py index b534897f5..2e357febe 100644 --- a/nova/tests/api/openstack/__init__.py +++ b/nova/tests/api/openstack/__init__.py @@ -27,11 +27,13 @@ class RateLimitingMiddlewareTest(unittest.TestCase): def test_get_action_name(self): middleware = RateLimitingMiddleware(APIStub()) + def verify(method, url, action_name): req = Request.blank(url) req.method = method action = middleware.get_action_name(req) self.assertEqual(action, action_name) + verify('PUT', '/servers/4', 'PUT') verify('DELETE', '/servers/4', 'DELETE') verify('POST', '/images/4', 'POST') @@ -60,7 +62,7 @@ class RateLimitingMiddlewareTest(unittest.TestCase): middleware = RateLimitingMiddleware(APIStub()) self.exhaust(middleware, 'POST', '/servers/4', 'usr1', 10) self.exhaust(middleware, 'POST', '/images/4', 'usr2', 10) - self.assertTrue(set(middleware.limiter._levels) == + self.assertTrue(set(middleware.limiter._levels) == set(['usr1:POST', 'usr1:POST servers', 'usr2:POST'])) def test_POST_servers_action_correctly_ratelimited(self): @@ -85,19 +87,19 @@ class LimiterTest(unittest.TestCase): def test_limiter(self): items = range(2000) req = Request.blank('/') - self.assertEqual(limited(items, req), items[ :1000]) + self.assertEqual(limited(items, req), items[:1000]) req = Request.blank('/?offset=0') - self.assertEqual(limited(items, req), items[ :1000]) + self.assertEqual(limited(items, req), items[:1000]) req = Request.blank('/?offset=3') self.assertEqual(limited(items, req), items[3:1003]) req = Request.blank('/?offset=2005') self.assertEqual(limited(items, req), []) req = Request.blank('/?limit=10') - self.assertEqual(limited(items, req), items[ :10]) + self.assertEqual(limited(items, req), items[:10]) req = Request.blank('/?limit=0') - self.assertEqual(limited(items, req), items[ :1000]) + self.assertEqual(limited(items, req), items[:1000]) req = Request.blank('/?limit=3000') - self.assertEqual(limited(items, req), items[ :1000]) + self.assertEqual(limited(items, req), items[:1000]) req = Request.blank('/?offset=1&limit=3') self.assertEqual(limited(items, req), items[1:4]) req = Request.blank('/?offset=3&limit=0') diff --git a/nova/tests/api/openstack/fakes.py b/nova/tests/api/openstack/fakes.py index 14170fbb2..7ecb72ab3 100644 --- a/nova/tests/api/openstack/fakes.py +++ b/nova/tests/api/openstack/fakes.py @@ -36,7 +36,7 @@ from nova.wsgi import Router FLAGS = flags.FLAGS -class Context(object): +class Context(object): pass @@ -84,11 +84,11 @@ def stub_out_image_service(stubs): def stub_out_auth(stubs): def fake_auth_init(self, app): self.application = app - - stubs.Set(nova.api.openstack.AuthMiddleware, - '__init__', fake_auth_init) - stubs.Set(nova.api.openstack.AuthMiddleware, - '__call__', fake_wsgi) + + stubs.Set(nova.api.openstack.AuthMiddleware, + '__init__', fake_auth_init) + stubs.Set(nova.api.openstack.AuthMiddleware, + '__call__', fake_wsgi) def stub_out_rate_limiting(stubs): @@ -105,7 +105,7 @@ def stub_out_rate_limiting(stubs): def stub_out_networking(stubs): def get_my_ip(): - return '127.0.0.1' + return '127.0.0.1' stubs.Set(nova.utils, 'get_my_ip', get_my_ip) FLAGS.FAKE_subdomain = 'api' @@ -137,7 +137,6 @@ def stub_out_glance(stubs, initial_fixtures=[]): return id def fake_update_image_metadata(self, image_id, image_data): - f = self.fake_get_image_metadata(image_id) if not f: raise exc.NotFound @@ -145,7 +144,6 @@ def stub_out_glance(stubs, initial_fixtures=[]): f.update(image_data) def fake_delete_image_metadata(self, image_id): - f = self.fake_get_image_metadata(image_id) if not f: raise exc.NotFound @@ -164,9 +162,11 @@ def stub_out_glance(stubs, initial_fixtures=[]): fake_parallax_client.fake_get_image_metadata) stubs.Set(nova.image.services.glance.ParallaxClient, 'add_image_metadata', fake_parallax_client.fake_add_image_metadata) - stubs.Set(nova.image.services.glance.ParallaxClient, 'update_image_metadata', + stubs.Set(nova.image.services.glance.ParallaxClient, + 'update_image_metadata', fake_parallax_client.fake_update_image_metadata) - stubs.Set(nova.image.services.glance.ParallaxClient, 'delete_image_metadata', + stubs.Set(nova.image.services.glance.ParallaxClient, + 'delete_image_metadata', fake_parallax_client.fake_delete_image_metadata) stubs.Set(nova.image.services.glance.GlanceImageService, 'delete_all', fake_parallax_client.fake_delete_all) @@ -174,7 +174,7 @@ def stub_out_glance(stubs, initial_fixtures=[]): class FakeToken(object): def __init__(self, **kwargs): - for k,v in kwargs.iteritems(): + for k, v in kwargs.iteritems(): setattr(self, k, v) @@ -200,7 +200,7 @@ class FakeAuthDatabase(object): class FakeAuthManager(object): auth_data = {} - def add_user(self, key, user): + def add_user(self, key, user): FakeAuthManager.auth_data[key] = user def get_user(self, uid): diff --git a/nova/tests/api/openstack/test_auth.py b/nova/tests/api/openstack/test_auth.py index 61d17d7e8..b63da187f 100644 --- a/nova/tests/api/openstack/test_auth.py +++ b/nova/tests/api/openstack/test_auth.py @@ -1,3 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + import datetime import unittest @@ -11,7 +28,9 @@ import nova.auth.manager from nova import auth from nova.tests.api.openstack import fakes + class Test(unittest.TestCase): + def setUp(self): self.stubs = stubout.StubOutForTesting() self.stubs.Set(nova.api.openstack.auth.BasicApiAuthManager, @@ -42,7 +61,7 @@ class Test(unittest.TestCase): def test_authorize_token(self): f = fakes.FakeAuthManager() f.add_user('derp', nova.auth.manager.User(1, 'herp', None, None, None)) - + req = webob.Request.blank('/v1.0/') req.headers['X-Auth-User'] = 'herp' req.headers['X-Auth-Key'] = 'derp' @@ -63,14 +82,14 @@ class Test(unittest.TestCase): result = req.get_response(nova.api.API()) self.assertEqual(result.status, '200 OK') self.assertEqual(result.headers['X-Test-Success'], 'True') - + def test_token_expiry(self): self.destroy_called = False token_hash = 'bacon' def destroy_token_mock(meh, context, token): self.destroy_called = True - + def bad_token(meh, context, token_hash): return fakes.FakeToken( token_hash=token_hash, diff --git a/nova/tests/api/openstack/test_faults.py b/nova/tests/api/openstack/test_faults.py index 70a811469..fda2b5ede 100644 --- a/nova/tests/api/openstack/test_faults.py +++ b/nova/tests/api/openstack/test_faults.py @@ -1,3 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + import unittest import webob import webob.dec @@ -5,6 +22,7 @@ import webob.exc from nova.api.openstack import faults + class TestFaults(unittest.TestCase): def test_fault_parts(self): @@ -19,7 +37,7 @@ class TestFaults(unittest.TestCase): def test_retry_header(self): req = webob.Request.blank('/.xml') - exc = webob.exc.HTTPRequestEntityTooLarge(explanation='sorry', + exc = webob.exc.HTTPRequestEntityTooLarge(explanation='sorry', headers={'Retry-After': 4}) f = faults.Fault(exc) resp = req.get_response(f) diff --git a/nova/tests/api/openstack/test_images.py b/nova/tests/api/openstack/test_images.py index e156c0957..d61c3a99b 100644 --- a/nova/tests/api/openstack/test_images.py +++ b/nova/tests/api/openstack/test_images.py @@ -90,7 +90,7 @@ class BaseImageServiceTests(object): id = self.service.create(fixture) fixture['status'] = 'in progress' - + self.service.update(id, fixture) new_image_data = self.service.show(id) self.assertEquals('in progress', new_image_data['status']) @@ -121,7 +121,7 @@ class BaseImageServiceTests(object): num_images = len(self.service.index()) self.assertEquals(2, num_images, str(self.service.index())) - + self.service.delete(ids[0]) num_images = len(self.service.index()) @@ -135,7 +135,8 @@ class LocalImageServiceTest(unittest.TestCase, def setUp(self): self.stubs = stubout.StubOutForTesting() - self.service = utils.import_object('nova.image.service.LocalImageService') + service_class = 'nova.image.service.LocalImageService' + self.service = utils.import_object(service_class) def tearDown(self): self.service.delete_all() @@ -150,7 +151,8 @@ class GlanceImageServiceTest(unittest.TestCase, def setUp(self): self.stubs = stubout.StubOutForTesting() fakes.stub_out_glance(self.stubs) - self.service = utils.import_object('nova.image.services.glance.GlanceImageService') + service_class = 'nova.image.services.glance.GlanceImageService' + self.service = utils.import_object(service_class) self.service.delete_all() def tearDown(self): @@ -172,8 +174,7 @@ class ImageControllerWithGlanceServiceTest(unittest.TestCase): 'deleted': False, 'is_public': True, 'status': 'available', - 'image_type': 'kernel' - }, + 'image_type': 'kernel'}, {'id': 'slkduhfas73kkaskgdas', 'name': 'public image #2', 'created_at': str(datetime.datetime.utcnow()), @@ -182,9 +183,7 @@ class ImageControllerWithGlanceServiceTest(unittest.TestCase): 'deleted': False, 'is_public': True, 'status': 'available', - 'image_type': 'ramdisk' - }, - ] + 'image_type': 'ramdisk'}] def setUp(self): self.orig_image_service = FLAGS.image_service @@ -211,7 +210,8 @@ class ImageControllerWithGlanceServiceTest(unittest.TestCase): in self.IMAGE_FIXTURES] for image in res_dict['images']: - self.assertEquals(1, fixture_index.count(image), "image %s not in fixture index!" % str(image)) + self.assertEquals(1, fixture_index.count(image), + "image %s not in fixture index!" % str(image)) def test_get_image_details(self): req = webob.Request.blank('/v1.0/images/detail') @@ -219,4 +219,5 @@ class ImageControllerWithGlanceServiceTest(unittest.TestCase): res_dict = json.loads(res.body) for image in res_dict['images']: - self.assertEquals(1, self.IMAGE_FIXTURES.count(image), "image %s not in fixtures!" % str(image)) + self.assertEquals(1, self.IMAGE_FIXTURES.count(image), + "image %s not in fixtures!" % str(image)) diff --git a/nova/tests/api/openstack/test_ratelimiting.py b/nova/tests/api/openstack/test_ratelimiting.py index ad9e67454..4c9d6bc23 100644 --- a/nova/tests/api/openstack/test_ratelimiting.py +++ b/nova/tests/api/openstack/test_ratelimiting.py @@ -6,6 +6,7 @@ import webob import nova.api.openstack.ratelimiting as ratelimiting + class LimiterTest(unittest.TestCase): def setUp(self): @@ -66,13 +67,16 @@ class LimiterTest(unittest.TestCase): class FakeLimiter(object): """Fake Limiter class that you can tell how to behave.""" + def __init__(self, test): self._action = self._username = self._delay = None self.test = test + def mock(self, action, username, delay): self._action = action self._username = username self._delay = delay + def perform(self, action, username): self.test.assertEqual(action, self._action) self.test.assertEqual(username, self._username) @@ -88,7 +92,7 @@ class WSGIAppTest(unittest.TestCase): def test_invalid_methods(self): requests = [] for method in ['GET', 'PUT', 'DELETE']: - req = webob.Request.blank('/limits/michael/breakdance', + req = webob.Request.blank('/limits/michael/breakdance', dict(REQUEST_METHOD=method)) requests.append(req) for req in requests: @@ -180,7 +184,7 @@ def wire_HTTPConnection_to_WSGI(host, app): the connection object will be a fake. Its requests will be sent directly to the given WSGI app rather than through a socket. - + Code connecting to hosts other than host will not be affected. This method may be called multiple times to map different hosts to @@ -189,13 +193,16 @@ def wire_HTTPConnection_to_WSGI(host, app): class HTTPConnectionDecorator(object): """Wraps the real HTTPConnection class so that when you instantiate the class you might instead get a fake instance.""" + def __init__(self, wrapped): self.wrapped = wrapped + def __call__(self, connection_host, *args, **kwargs): if connection_host == host: return FakeHttplibConnection(app, host) else: return self.wrapped(connection_host, *args, **kwargs) + httplib.HTTPConnection = HTTPConnectionDecorator(httplib.HTTPConnection) diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index d1ee533b6..55efcf733 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -32,9 +32,9 @@ from nova.tests.api.openstack import fakes FLAGS = flags.FLAGS - FLAGS.verbose = True + def return_server(context, id): return stub_instance(id) @@ -44,10 +44,8 @@ def return_servers(context, user_id=1): def stub_instance(id, user_id=1): - return Instance( - id=id, state=0, image_id=10, server_name='server%s'%id, - user_id=user_id - ) + return Instance(id=id, state=0, image_id=10, server_name='server%s' % id, + user_id=user_id) class ServersTest(unittest.TestCase): @@ -61,9 +59,10 @@ class ServersTest(unittest.TestCase): fakes.stub_out_key_pair_funcs(self.stubs) fakes.stub_out_image_service(self.stubs) self.stubs.Set(nova.db.api, 'instance_get_all', return_servers) - self.stubs.Set(nova.db.api, 'instance_get_by_internal_id', return_server) - self.stubs.Set(nova.db.api, 'instance_get_all_by_user', - return_servers) + self.stubs.Set(nova.db.api, 'instance_get_by_internal_id', + return_server) + self.stubs.Set(nova.db.api, 'instance_get_all_by_user', + return_servers) def tearDown(self): self.stubs.UnsetAll() @@ -79,17 +78,17 @@ class ServersTest(unittest.TestCase): req = webob.Request.blank('/v1.0/servers') res = req.get_response(nova.api.API()) res_dict = json.loads(res.body) - + i = 0 for s in res_dict['servers']: self.assertEqual(s['id'], i) - self.assertEqual(s['name'], 'server%d'%i) + self.assertEqual(s['name'], 'server%d' % i) self.assertEqual(s.get('imageId', None), None) i += 1 def test_create_instance(self): def server_update(context, id, params): - pass + pass def instance_create(context, inst): class Foo(object): @@ -98,9 +97,9 @@ class ServersTest(unittest.TestCase): def fake_method(*args, **kwargs): pass - + def project_get_network(context, user_id): - return dict(id='1', host='localhost') + return dict(id='1', host='localhost') def queue_get_for(context, *args): return 'network_topic' @@ -114,11 +113,10 @@ class ServersTest(unittest.TestCase): self.stubs.Set(nova.db.api, 'queue_get_for', queue_get_for) self.stubs.Set(nova.network.manager.VlanManager, 'allocate_fixed_ip', fake_method) - + body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, - personality = {} - )) + personality={})) req = webob.Request.blank('/v1.0/servers') req.method = 'POST' req.body = json.dumps(body) @@ -188,44 +186,41 @@ class ServersTest(unittest.TestCase): req = webob.Request.blank('/v1.0/servers/detail') res = req.get_response(nova.api.API()) res_dict = json.loads(res.body) - + i = 0 for s in res_dict['servers']: self.assertEqual(s['id'], i) - self.assertEqual(s['name'], 'server%d'%i) + self.assertEqual(s['name'], 'server%d' % i) self.assertEqual(s['imageId'], 10) i += 1 def test_server_reboot(self): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, - personality = {} - )) + personality={})) req = webob.Request.blank('/v1.0/servers/1/action') req.method = 'POST' - req.content_type= 'application/json' + req.content_type = 'application/json' req.body = json.dumps(body) res = req.get_response(nova.api.API()) def test_server_rebuild(self): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, - personality = {} - )) + personality={})) req = webob.Request.blank('/v1.0/servers/1/action') req.method = 'POST' - req.content_type= 'application/json' + req.content_type = 'application/json' req.body = json.dumps(body) res = req.get_response(nova.api.API()) def test_server_resize(self): body = dict(server=dict( name='server_test', imageId=2, flavorId=2, metadata={}, - personality = {} - )) + personality={})) req = webob.Request.blank('/v1.0/servers/1/action') req.method = 'POST' - req.content_type= 'application/json' + req.content_type = 'application/json' req.body = json.dumps(body) res = req.get_response(nova.api.API()) @@ -234,8 +229,9 @@ class ServersTest(unittest.TestCase): req.method = 'DELETE' self.server_delete_called = False + def instance_destroy_mock(context, id): - self.server_delete_called = True + self.server_delete_called = True self.stubs.Set(nova.db.api, 'instance_destroy', instance_destroy_mock) diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index 9425b01d0..604d0cb66 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -72,7 +72,7 @@ class Test(unittest.TestCase): """Test controller to call from router.""" test = self - def show(self, req, id): # pylint: disable-msg=W0622,C0103 + def show(self, req, id): # pylint: disable-msg=W0622,C0103 """Default action called for requests with an ID.""" self.test.assertEqual(req.path_info, '/tests/123') self.test.assertEqual(id, '123') @@ -95,7 +95,7 @@ class Test(unittest.TestCase): class SerializerTest(unittest.TestCase): def match(self, url, accept, expect): - input_dict = dict(servers=dict(a=(2,3))) + input_dict = dict(servers=dict(a=(2, 3))) expected_xml = '(2,3)' expected_json = '{"servers":{"a":[2,3]}}' req = webob.Request.blank(url, headers=dict(Accept=accept)) diff --git a/nova/tests/api_integration.py b/nova/tests/api_integration.py index 23a88f083..54403c655 100644 --- a/nova/tests/api_integration.py +++ b/nova/tests/api_integration.py @@ -28,16 +28,17 @@ CLC_IP = '127.0.0.1' CLC_PORT = 8773 REGION = 'test' + def get_connection(): - return boto.connect_ec2 ( + return boto.connect_ec2( aws_access_key_id=ACCESS_KEY, aws_secret_access_key=SECRET_KEY, is_secure=False, region=RegionInfo(None, REGION, CLC_IP), port=CLC_PORT, path='/services/Cloud', - debug=99 - ) + debug=99) + class APIIntegrationTests(unittest.TestCase): def test_001_get_all_images(self): @@ -51,4 +52,3 @@ if __name__ == '__main__': #print conn.get_all_key_pairs() #print conn.create_key_pair #print conn.create_security_group('name', 'description') - diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 55a7ff32f..80493b10a 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -99,6 +99,7 @@ class XmlConversionTestCase(test.BaseTestCase): self.assertEqual(conv('-'), '-') self.assertEqual(conv('-0'), 0) + class ApiEc2TestCase(test.BaseTestCase): """Unit test for the cloud controller on an EC2 API""" def setUp(self): @@ -138,7 +139,6 @@ class ApiEc2TestCase(test.BaseTestCase): self.manager.delete_project(project) self.manager.delete_user(user) - def test_get_all_key_pairs(self): """Test that, after creating a user and project and generating a key pair, that the API call to list key pairs works properly""" @@ -183,7 +183,7 @@ class ApiEc2TestCase(test.BaseTestCase): self.manager.add_role('fake', 'netadmin') project.add_role('fake', 'netadmin') - security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") \ + security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") for x in range(random.randint(4, 8))) self.ec2.create_security_group(security_group_name, 'test group') @@ -217,10 +217,11 @@ class ApiEc2TestCase(test.BaseTestCase): self.manager.add_role('fake', 'netadmin') project.add_role('fake', 'netadmin') - security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") \ + security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") for x in range(random.randint(4, 8))) - group = self.ec2.create_security_group(security_group_name, 'test group') + group = self.ec2.create_security_group(security_group_name, + 'test group') self.expect_http() self.mox.ReplayAll() @@ -282,12 +283,14 @@ class ApiEc2TestCase(test.BaseTestCase): self.manager.add_role('fake', 'netadmin') project.add_role('fake', 'netadmin') - security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") \ + rand_string = 'sdiuisudfsdcnpaqwertasd' + security_group_name = "".join(random.choice(rand_string) + for x in range(random.randint(4, 8))) + other_security_group_name = "".join(random.choice(rand_string) for x in range(random.randint(4, 8))) - other_security_group_name = "".join(random.choice("sdiuisudfsdcnpaqwertasd") \ - for x in range(random.randint(4, 8))) - group = self.ec2.create_security_group(security_group_name, 'test group') + group = self.ec2.create_security_group(security_group_name, + 'test group') self.expect_http() self.mox.ReplayAll() @@ -313,9 +316,8 @@ class ApiEc2TestCase(test.BaseTestCase): if group.name == security_group_name: self.assertEquals(len(group.rules), 1) self.assertEquals(len(group.rules[0].grants), 1) - self.assertEquals(str(group.rules[0].grants[0]), - '%s-%s' % (other_security_group_name, 'fake')) - + self.assertEquals(str(group.rules[0].grants[0]), '%s-%s' % + (other_security_group_name, 'fake')) self.expect_http() self.mox.ReplayAll() diff --git a/nova/tests/auth_unittest.py b/nova/tests/auth_unittest.py index 97d22d702..fe891beee 100644 --- a/nova/tests/auth_unittest.py +++ b/nova/tests/auth_unittest.py @@ -28,6 +28,7 @@ from nova.api.ec2 import cloud FLAGS = flags.FLAGS + class user_generator(object): def __init__(self, manager, **user_state): if 'name' not in user_state: @@ -41,6 +42,7 @@ class user_generator(object): def __exit__(self, value, type, trace): self.manager.delete_user(self.user) + class project_generator(object): def __init__(self, manager, **project_state): if 'name' not in project_state: @@ -56,6 +58,7 @@ class project_generator(object): def __exit__(self, value, type, trace): self.manager.delete_project(self.project) + class user_and_project_generator(object): def __init__(self, manager, user_state={}, project_state={}): self.manager = manager @@ -75,6 +78,7 @@ class user_and_project_generator(object): self.manager.delete_user(self.user) self.manager.delete_project(self.project) + class AuthManagerTestCase(object): def setUp(self): FLAGS.auth_driver = self.auth_driver @@ -96,7 +100,7 @@ class AuthManagerTestCase(object): self.assertEqual('private-party', u.access) def test_004_signature_is_valid(self): - #self.assertTrue(self.manager.authenticate( **boto.generate_url ... ? ? ? )) + #self.assertTrue(self.manager.authenticate(**boto.generate_url ...? )) pass #raise NotImplementedError @@ -127,7 +131,7 @@ class AuthManagerTestCase(object): self.assertFalse(self.manager.has_role('test1', 'itsec')) def test_can_create_and_get_project(self): - with user_and_project_generator(self.manager) as (u,p): + with user_and_project_generator(self.manager) as (u, p): self.assert_(self.manager.get_user('test1')) self.assert_(self.manager.get_user('test1')) self.assert_(self.manager.get_project('testproj')) @@ -321,6 +325,7 @@ class AuthManagerTestCase(object): self.assertEqual('secret', user.secret) self.assertTrue(user.is_admin()) + class AuthManagerLdapTestCase(AuthManagerTestCase, test.TrialTestCase): auth_driver = 'nova.auth.ldapdriver.FakeLdapDriver' @@ -337,6 +342,7 @@ class AuthManagerLdapTestCase(AuthManagerTestCase, test.TrialTestCase): except: self.skip = True + class AuthManagerDbTestCase(AuthManagerTestCase, test.TrialTestCase): auth_driver = 'nova.auth.dbdriver.DbDriver' diff --git a/nova/tests/cloud_unittest.py b/nova/tests/cloud_unittest.py index 835bfdf49..2d61d2675 100644 --- a/nova/tests/cloud_unittest.py +++ b/nova/tests/cloud_unittest.py @@ -46,13 +46,13 @@ from nova.objectstore import image FLAGS = flags.FLAGS - # Temp dirs for working with image attributes through the cloud controller # (stole this from objectstore_unittest.py) OSS_TEMPDIR = tempfile.mkdtemp(prefix='test_oss-') IMAGES_PATH = os.path.join(OSS_TEMPDIR, 'images') os.makedirs(IMAGES_PATH) + class CloudTestCase(test.TrialTestCase): def setUp(self): super(CloudTestCase, self).setUp() @@ -97,17 +97,17 @@ class CloudTestCase(test.TrialTestCase): max_count = 1 kwargs = {'image_id': image_id, 'instance_type': instance_type, - 'max_count': max_count } + 'max_count': max_count} rv = yield self.cloud.run_instances(self.context, **kwargs) instance_id = rv['instancesSet'][0]['instanceId'] - output = yield self.cloud.get_console_output(context=self.context, instance_id=[instance_id]) + output = yield self.cloud.get_console_output(context=self.context, + instance_id=[instance_id]) self.assertEquals(b64decode(output['output']), 'FAKE CONSOLE OUTPUT') # TODO(soren): We need this until we can stop polling in the rpc code # for unit tests. greenthread.sleep(0.3) rv = yield self.cloud.terminate_instances(self.context, [instance_id]) - def test_key_generation(self): result = self._create_key('test') private_key = result['private_key'] @@ -146,8 +146,10 @@ class CloudTestCase(test.TrialTestCase): 'max_count': max_count} rv = yield self.cloud.run_instances(self.context, **kwargs) # TODO: check for proper response - instance = rv['reservationSet'][0][rv['reservationSet'][0].keys()[0]][0] - logging.debug("Need to watch instance %s until it's running..." % instance['instance_id']) + instance_id = rv['reservationSet'][0].keys()[0] + instance = rv['reservationSet'][0][instance_id][0] + logging.debug("Need to watch instance %s until it's running..." % + instance['instance_id']) while True: rv = yield defer.succeed(time.sleep(1)) info = self.cloud._get_instance(instance['instance_id']) @@ -157,14 +159,15 @@ class CloudTestCase(test.TrialTestCase): self.assert_(rv) if connection_type != 'fake': - time.sleep(45) # Should use boto for polling here + time.sleep(45) # Should use boto for polling here for reservations in rv['reservationSet']: # for res_id in reservations.keys(): - # logging.debug(reservations[res_id]) - # for instance in reservations[res_id]: - for instance in reservations[reservations.keys()[0]]: - logging.debug("Terminating instance %s" % instance['instance_id']) - rv = yield self.compute.terminate_instance(instance['instance_id']) + # logging.debug(reservations[res_id]) + # for instance in reservations[res_id]: + for instance in reservations[reservations.keys()[0]]: + instance_id = instance['instance_id'] + logging.debug("Terminating instance %s" % instance_id) + rv = yield self.compute.terminate_instance(instance_id) def test_instance_update_state(self): def instance(num): @@ -183,8 +186,7 @@ class CloudTestCase(test.TrialTestCase): 'groups': ['default'], 'product_codes': None, 'state': 0x01, - 'user_data': '' - } + 'user_data': ''} rv = self.cloud._format_describe_instances(self.context) self.assert_(len(rv['reservationSet']) == 0) @@ -199,7 +201,9 @@ class CloudTestCase(test.TrialTestCase): #self.assert_(len(rv['reservationSet'][0]['instances_set']) == 5) # report 4 nodes each having 1 of the instances #for i in xrange(4): - # self.cloud.update_state('instances', {('node-%s' % i): {('i-%s' % i): instance(i)}}) + # self.cloud.update_state('instances', + # {('node-%s' % i): {('i-%s' % i): + # instance(i)}}) # one instance should be pending still #self.assert_(len(self.cloud.instances['pending'].keys()) == 1) @@ -217,8 +221,10 @@ class CloudTestCase(test.TrialTestCase): @staticmethod def _fake_set_image_description(ctxt, image_id, description): from nova.objectstore import handler + class req: pass + request = req() request.context = ctxt request.args = {'image_id': [image_id], diff --git a/nova/tests/flags_unittest.py b/nova/tests/flags_unittest.py index 714170e5e..b97df075d 100644 --- a/nova/tests/flags_unittest.py +++ b/nova/tests/flags_unittest.py @@ -23,7 +23,9 @@ from nova import test FLAGS = flags.FLAGS flags.DEFINE_string('flags_unittest', 'foo', 'for testing purposes only') + class FlagsTestCase(test.TrialTestCase): + def setUp(self): super(FlagsTestCase, self).setUp() self.FLAGS = flags.FlagValues() @@ -35,7 +37,8 @@ class FlagsTestCase(test.TrialTestCase): self.assert_('false' not in self.FLAGS) self.assert_('true' not in self.FLAGS) - flags.DEFINE_string('string', 'default', 'desc', flag_values=self.FLAGS) + flags.DEFINE_string('string', 'default', 'desc', + flag_values=self.FLAGS) flags.DEFINE_integer('int', 1, 'desc', flag_values=self.FLAGS) flags.DEFINE_bool('false', False, 'desc', flag_values=self.FLAGS) flags.DEFINE_bool('true', True, 'desc', flag_values=self.FLAGS) diff --git a/nova/tests/network_unittest.py b/nova/tests/network_unittest.py index 44cf47407..b7caed4fd 100644 --- a/nova/tests/network_unittest.py +++ b/nova/tests/network_unittest.py @@ -98,7 +98,6 @@ class NetworkTestCase(test.TrialTestCase): self.context.project_id = self.projects[project_num].id self.network.deallocate_fixed_ip(self.context, address) - def test_public_network_association(self): """Makes sure that we can allocaate a public ip""" # TODO(vish): better way of adding floating ips @@ -118,10 +117,12 @@ class NetworkTestCase(test.TrialTestCase): lease_ip(fix_addr) self.assertEqual(float_addr, str(pubnet[0])) self.network.associate_floating_ip(self.context, float_addr, fix_addr) - address = db.instance_get_floating_address(context.get_admin_context(), self.instance_id) + address = db.instance_get_floating_address(context.get_admin_context(), + self.instance_id) self.assertEqual(address, float_addr) self.network.disassociate_floating_ip(self.context, float_addr) - address = db.instance_get_floating_address(context.get_admin_context(), self.instance_id) + address = db.instance_get_floating_address(context.get_admin_context(), + self.instance_id) self.assertEqual(address, None) self.network.deallocate_floating_ip(self.context, float_addr) self.network.deallocate_fixed_ip(self.context, fix_addr) @@ -254,18 +255,24 @@ class NetworkTestCase(test.TrialTestCase): There are ips reserved at the bottom and top of the range. services (network, gateway, CloudPipe, broadcast) """ - network = db.project_get_network(context.get_admin_context(), self.projects[0].id) + network = db.project_get_network(context.get_admin_context(), + self.projects[0].id) net_size = flags.FLAGS.network_size - total_ips = (db.network_count_available_ips(context.get_admin_context(), network['id']) + - db.network_count_reserved_ips(context.get_admin_context(), network['id']) + - db.network_count_allocated_ips(context.get_admin_context(), network['id'])) + admin_context = context.get_admin_context() + total_ips = (db.network_count_available_ips(admin_context, + network['id']) + + db.network_count_reserved_ips(admin_context, + network['id']) + + db.network_count_allocated_ips(admin_context, + network['id'])) self.assertEqual(total_ips, net_size) def test_too_many_addresses(self): """Test for a NoMoreAddresses exception when all fixed ips are used. """ - network = db.project_get_network(context.get_admin_context(), self.projects[0].id) - num_available_ips = db.network_count_available_ips(context.get_admin_context(), + admin_context = context.get_admin_context() + network = db.project_get_network(admin_context, self.projects[0].id) + num_available_ips = db.network_count_available_ips(admin_context, network['id']) addresses = [] instance_ids = [] @@ -276,8 +283,9 @@ class NetworkTestCase(test.TrialTestCase): addresses.append(address) lease_ip(address) - self.assertEqual(db.network_count_available_ips(context.get_admin_context(), - network['id']), 0) + ip_count = db.network_count_available_ips(context.get_admin_context(), + network['id']) + self.assertEqual(ip_count, 0) self.assertRaises(db.NoMoreAddresses, self.network.allocate_fixed_ip, self.context, @@ -287,14 +295,15 @@ class NetworkTestCase(test.TrialTestCase): self.network.deallocate_fixed_ip(self.context, addresses[i]) release_ip(addresses[i]) db.instance_destroy(context.get_admin_context(), instance_ids[i]) - self.assertEqual(db.network_count_available_ips(context.get_admin_context(), - network['id']), - num_available_ips) + ip_count = db.network_count_available_ips(context.get_admin_context(), + network['id']) + self.assertEqual(ip_count, num_available_ips) def is_allocated_in_project(address, project_id): """Returns true if address is in specified project""" - project_net = db.project_get_network(context.get_admin_context(), project_id) + project_net = db.project_get_network(context.get_admin_context(), + project_id) network = db.fixed_ip_get_network(context.get_admin_context(), address) instance = db.fixed_ip_get_instance(context.get_admin_context(), address) # instance exists until release @@ -308,8 +317,10 @@ def binpath(script): def lease_ip(private_ip): """Run add command on dhcpbridge""" - network_ref = db.fixed_ip_get_network(context.get_admin_context(), private_ip) - instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) + network_ref = db.fixed_ip_get_network(context.get_admin_context(), + private_ip) + instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), + private_ip) cmd = "%s add %s %s fake" % (binpath('nova-dhcpbridge'), instance_ref['mac_address'], private_ip) @@ -322,8 +333,10 @@ def lease_ip(private_ip): def release_ip(private_ip): """Run del command on dhcpbridge""" - network_ref = db.fixed_ip_get_network(context.get_admin_context(), private_ip) - instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) + network_ref = db.fixed_ip_get_network(context.get_admin_context(), + private_ip) + instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), + private_ip) cmd = "%s del %s %s fake" % (binpath('nova-dhcpbridge'), instance_ref['mac_address'], private_ip) diff --git a/nova/tests/objectstore_unittest.py b/nova/tests/objectstore_unittest.py index d1604c4e5..061799923 100644 --- a/nova/tests/objectstore_unittest.py +++ b/nova/tests/objectstore_unittest.py @@ -181,7 +181,7 @@ class ObjectStoreTestCase(test.TrialTestCase): class TestHTTPChannel(http.HTTPChannel): """Dummy site required for twisted.web""" - def checkPersistence(self, _, __): # pylint: disable-msg=C0103 + def checkPersistence(self, _, __): # pylint: disable-msg=C0103 """Otherwise we end up with an unclean reactor.""" return False @@ -217,7 +217,6 @@ class S3APITestCase(test.TrialTestCase): # pylint: enable-msg=E1101 self.tcp_port = self.listening_port.getHost().port - if not boto.config.has_section('Boto'): boto.config.add_section('Boto') boto.config.set('Boto', 'num_retries', '0') @@ -234,11 +233,11 @@ class S3APITestCase(test.TrialTestCase): self.conn.get_http_connection = get_http_connection - def _ensure_no_buckets(self, buckets): # pylint: disable-msg=C0111 + def _ensure_no_buckets(self, buckets): # pylint: disable-msg=C0111 self.assertEquals(len(buckets), 0, "Bucket list was not empty") return True - def _ensure_one_bucket(self, buckets, name): # pylint: disable-msg=C0111 + def _ensure_one_bucket(self, buckets, name): # pylint: disable-msg=C0111 self.assertEquals(len(buckets), 1, "Bucket list didn't have exactly one element in it") self.assertEquals(buckets[0].name, name, "Wrong name") diff --git a/nova/tests/process_unittest.py b/nova/tests/process_unittest.py index 25c60c616..67245af03 100644 --- a/nova/tests/process_unittest.py +++ b/nova/tests/process_unittest.py @@ -38,6 +38,7 @@ class ProcessTestCase(test.TrialTestCase): def test_execute_stdout(self): pool = process.ProcessPool(2) d = pool.simple_execute('echo test') + def _check(rv): self.assertEqual(rv[0], 'test\n') self.assertEqual(rv[1], '') @@ -49,6 +50,7 @@ class ProcessTestCase(test.TrialTestCase): def test_execute_stderr(self): pool = process.ProcessPool(2) d = pool.simple_execute('cat BAD_FILE', check_exit_code=False) + def _check(rv): self.assertEqual(rv[0], '') self.assert_('No such file' in rv[1]) @@ -72,6 +74,7 @@ class ProcessTestCase(test.TrialTestCase): d4 = pool.simple_execute('sleep 0.005') called = [] + def _called(rv, name): called.append(name) diff --git a/nova/tests/quota_unittest.py b/nova/tests/quota_unittest.py index 92f3be508..9e3afbf4e 100644 --- a/nova/tests/quota_unittest.py +++ b/nova/tests/quota_unittest.py @@ -141,12 +141,13 @@ class QuotaTestCase(test.TrialTestCase): try: db.floating_ip_get_by_address(context.get_admin_context(), address) except exception.NotFound: - db.floating_ip_create(context.get_admin_context(), {'address': address, - 'host': FLAGS.host}) + db.floating_ip_create(context.get_admin_context(), + {'address': address, 'host': FLAGS.host}) float_addr = self.network.allocate_floating_ip(self.context, self.project.id) # NOTE(vish): This assert never fails. When cloud attempts to # make an rpc.call, the test just finishes with OK. It # appears to be something in the magic inline callbacks # that is breaking. - self.assertRaises(cloud.QuotaError, self.cloud.allocate_address, self.context) + self.assertRaises(cloud.QuotaError, self.cloud.allocate_address, + self.context) diff --git a/nova/tests/rpc_unittest.py b/nova/tests/rpc_unittest.py index c380393a0..f35b65a39 100644 --- a/nova/tests/rpc_unittest.py +++ b/nova/tests/rpc_unittest.py @@ -41,7 +41,7 @@ class RpcTestCase(test.TrialTestCase): topic='test', proxy=self.receiver) self.consumer.attach_to_twisted() - self.context= context.get_admin_context() + self.context = context.get_admin_context() def test_call_succeed(self): """Get a value through rpc call""" @@ -67,9 +67,9 @@ class RpcTestCase(test.TrialTestCase): to an int in the test. """ value = 42 - self.assertFailure(rpc.call_twisted(self.context, - 'test', {"method": "fail", - "args": {"value": value}}), + self.assertFailure(rpc.call_twisted(self.context, 'test', + {"method": "fail", + "args": {"value": value}}), rpc.RemoteError) try: yield rpc.call_twisted(self.context, @@ -101,4 +101,3 @@ class TestReceiver(object): def fail(context, value): """Raises an exception with the value sent in""" raise Exception(value) - diff --git a/nova/tests/scheduler_unittest.py b/nova/tests/scheduler_unittest.py index 166a32702..27345d055 100644 --- a/nova/tests/scheduler_unittest.py +++ b/nova/tests/scheduler_unittest.py @@ -34,6 +34,7 @@ from nova.scheduler import driver FLAGS = flags.FLAGS flags.DECLARE('max_cores', 'nova.scheduler.simple') + class TestDriver(driver.Scheduler): """Scheduler Driver for Tests""" def schedule(context, topic, *args, **kwargs): @@ -42,6 +43,7 @@ class TestDriver(driver.Scheduler): def schedule_named_method(context, topic, num): return 'named_host' + class SchedulerTestCase(test.TrialTestCase): """Test case for scheduler""" def setUp(self): diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index 6df108a98..e74e0f726 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -179,7 +179,8 @@ class ServiceTestCase(test.BaseTestCase): binary).AndRaise(exception.NotFound()) service.db.service_create(self.context, service_create).AndReturn(service_ref) - service.db.service_get(self.context, service_ref['id']).AndReturn(service_ref) + service.db.service_get(self.context, + service_ref['id']).AndReturn(service_ref) service.db.service_update(self.context, service_ref['id'], mox.ContainsKeyValue('report_count', 1)) @@ -227,4 +228,3 @@ class ServiceTestCase(test.BaseTestCase): rv = yield s.report_state(host, binary) self.assert_(not s.model_disconnected) - diff --git a/nova/tests/validator_unittest.py b/nova/tests/validator_unittest.py index 84daa135e..b5f1c0667 100644 --- a/nova/tests/validator_unittest.py +++ b/nova/tests/validator_unittest.py @@ -35,7 +35,8 @@ class ValidationTestCase(test.TrialTestCase): self.assertTrue(type_case("foo", 5, 1)) self.assertRaises(TypeError, type_case, "bar", "5", 1) self.assertRaises(TypeError, type_case, None, 5, 1) - + + @validate.typetest(instanceid=str, size=int, number_of_instances=int) def type_case(instanceid, size, number_of_instances): return True diff --git a/nova/tests/virt_unittest.py b/nova/tests/virt_unittest.py index 76af5cabd..ce78d450c 100644 --- a/nova/tests/virt_unittest.py +++ b/nova/tests/virt_unittest.py @@ -29,11 +29,13 @@ from nova.virt import libvirt_conn FLAGS = flags.FLAGS flags.DECLARE('instances_path', 'nova.compute.manager') + class LibvirtConnTestCase(test.TrialTestCase): def setUp(self): super(LibvirtConnTestCase, self).setUp() self.manager = manager.AuthManager() - self.user = self.manager.create_user('fake', 'fake', 'fake', admin=True) + self.user = self.manager.create_user('fake', 'fake', 'fake', + admin=True) self.project = self.manager.create_project('fake', 'fake', 'fake') self.network = utils.import_object(FLAGS.network_manager) FLAGS.instances_path = '' @@ -41,15 +43,15 @@ class LibvirtConnTestCase(test.TrialTestCase): def test_get_uri_and_template(self): ip = '10.11.12.13' - instance = { 'internal_id' : 1, - 'memory_kb' : '1024000', - 'basepath' : '/some/path', - 'bridge_name' : 'br100', - 'mac_address' : '02:12:34:46:56:67', - 'vcpus' : 2, - 'project_id' : 'fake', - 'bridge' : 'br101', - 'instance_type' : 'm1.small'} + instance = {'internal_id': 1, + 'memory_kb': '1024000', + 'basepath': '/some/path', + 'bridge_name': 'br100', + 'mac_address': '02:12:34:46:56:67', + 'vcpus': 2, + 'project_id': 'fake', + 'bridge': 'br101', + 'instance_type': 'm1.small'} user_context = context.RequestContext(project=self.project, user=self.user) @@ -58,36 +60,34 @@ class LibvirtConnTestCase(test.TrialTestCase): self.network.set_network_host(context.get_admin_context(), network_ref['id']) - fixed_ip = { 'address' : ip, - 'network_id' : network_ref['id'] } + fixed_ip = {'address': ip, + 'network_id': network_ref['id']} ctxt = context.get_admin_context() fixed_ip_ref = db.fixed_ip_create(ctxt, fixed_ip) db.fixed_ip_update(ctxt, ip, {'allocated': True, - 'instance_id': instance_ref['id'] }) - - type_uri_map = { 'qemu' : ('qemu:///system', - [(lambda t: t.find('.').get('type'), 'qemu'), - (lambda t: t.find('./os/type').text, 'hvm'), - (lambda t: t.find('./devices/emulator'), None)]), - 'kvm' : ('qemu:///system', - [(lambda t: t.find('.').get('type'), 'kvm'), - (lambda t: t.find('./os/type').text, 'hvm'), - (lambda t: t.find('./devices/emulator'), None)]), - 'uml' : ('uml:///system', - [(lambda t: t.find('.').get('type'), 'uml'), - (lambda t: t.find('./os/type').text, 'uml')]), - } - - common_checks = [(lambda t: t.find('.').tag, 'domain'), - (lambda t: \ - t.find('./devices/interface/filterref/parameter') \ - .get('name'), 'IP'), - (lambda t: \ - t.find('./devices/interface/filterref/parameter') \ - .get('value'), '10.11.12.13')] - - for (libvirt_type,(expected_uri, checks)) in type_uri_map.iteritems(): + 'instance_id': instance_ref['id']}) + + type_uri_map = {'qemu': ('qemu:///system', + [(lambda t: t.find('.').get('type'), 'qemu'), + (lambda t: t.find('./os/type').text, 'hvm'), + (lambda t: t.find('./devices/emulator'), None)]), + 'kvm': ('qemu:///system', + [(lambda t: t.find('.').get('type'), 'kvm'), + (lambda t: t.find('./os/type').text, 'hvm'), + (lambda t: t.find('./devices/emulator'), None)]), + 'uml': ('uml:///system', + [(lambda t: t.find('.').get('type'), 'uml'), + (lambda t: t.find('./os/type').text, 'uml')])} + + common_checks = [ + (lambda t: t.find('.').tag, 'domain'), + (lambda t: t.find('./devices/interface/filterref/parameter').\ + get('name'), 'IP'), + (lambda t: t.find('./devices/interface/filterref/parameter').\ + get('value'), '10.11.12.13')] + + for (libvirt_type, (expected_uri, checks)) in type_uri_map.iteritems(): FLAGS.libvirt_type = libvirt_type conn = libvirt_conn.LibvirtConnection(True) @@ -111,19 +111,20 @@ class LibvirtConnTestCase(test.TrialTestCase): # implementation doesn't fiddle around with the FLAGS. testuri = 'something completely different' FLAGS.libvirt_uri = testuri - for (libvirt_type,(expected_uri, checks)) in type_uri_map.iteritems(): + for (libvirt_type, (expected_uri, checks)) in type_uri_map.iteritems(): FLAGS.libvirt_type = libvirt_type conn = libvirt_conn.LibvirtConnection(True) uri, template = conn.get_uri_and_template() self.assertEquals(uri, testuri) - def tearDown(self): super(LibvirtConnTestCase, self).tearDown() self.manager.delete_project(self.project) self.manager.delete_user(self.user) + class NWFilterTestCase(test.TrialTestCase): + def setUp(self): super(NWFilterTestCase, self).setUp() @@ -131,7 +132,8 @@ class NWFilterTestCase(test.TrialTestCase): pass self.manager = manager.AuthManager() - self.user = self.manager.create_user('fake', 'fake', 'fake', admin=True) + self.user = self.manager.create_user('fake', 'fake', 'fake', + admin=True) self.project = self.manager.create_project('fake', 'fake', 'fake') self.context = context.RequestContext(self.user, self.project) @@ -143,7 +145,6 @@ class NWFilterTestCase(test.TrialTestCase): self.manager.delete_project(self.project) self.manager.delete_user(self.user) - def test_cidr_rule_nwfilter_xml(self): cloud_controller = cloud.CloudController() cloud_controller.create_security_group(self.context, @@ -156,7 +157,6 @@ class NWFilterTestCase(test.TrialTestCase): ip_protocol='tcp', cidr_ip='0.0.0.0/0') - security_group = db.security_group_get_by_name(self.context, 'fake', 'testgroup') @@ -182,15 +182,12 @@ class NWFilterTestCase(test.TrialTestCase): self.assertEqual(ip_conditions[0].getAttribute('srcipmask'), '0.0.0.0') self.assertEqual(ip_conditions[0].getAttribute('dstportstart'), '80') self.assertEqual(ip_conditions[0].getAttribute('dstportend'), '81') - - self.teardown_security_group() def teardown_security_group(self): cloud_controller = cloud.CloudController() cloud_controller.delete_security_group(self.context, 'testgroup') - def setup_and_return_security_group(self): cloud_controller = cloud.CloudController() cloud_controller.create_security_group(self.context, @@ -244,16 +241,19 @@ class NWFilterTestCase(test.TrialTestCase): for required in [secgroup_filter, 'allow-dhcp-server', 'no-arp-spoofing', 'no-ip-spoofing', 'no-mac-spoofing']: - self.assertTrue(required in self.recursive_depends[instance_filter], - "Instance's filter does not include %s" % required) + self.assertTrue(required in + self.recursive_depends[instance_filter], + "Instance's filter does not include %s" % + required) self.security_group = self.setup_and_return_security_group() - db.instance_add_security_group(self.context, inst_id, self.security_group.id) + db.instance_add_security_group(self.context, inst_id, + self.security_group.id) instance = db.instance_get(self.context, inst_id) d = self.fw.setup_nwfilters_for_instance(instance) d.addCallback(_ensure_all_called) - d.addCallback(lambda _:self.teardown_security_group()) + d.addCallback(lambda _: self.teardown_security_group()) return d diff --git a/nova/tests/volume_unittest.py b/nova/tests/volume_unittest.py index 20bae8a7f..fdee30b48 100644 --- a/nova/tests/volume_unittest.py +++ b/nova/tests/volume_unittest.py @@ -59,7 +59,8 @@ class VolumeTestCase(test.TrialTestCase): """Test volume can be created and deleted""" volume_id = self._create_volume() yield self.volume.create_volume(self.context, volume_id) - self.assertEqual(volume_id, db.volume_get(context.get_admin_context(), volume_id).id) + self.assertEqual(volume_id, db.volume_get(context.get_admin_context(), + volume_id).id) yield self.volume.delete_volume(self.context, volume_id) self.assertRaises(exception.NotFound, @@ -114,7 +115,8 @@ class VolumeTestCase(test.TrialTestCase): volume_id = self._create_volume() yield self.volume.create_volume(self.context, volume_id) if FLAGS.fake_tests: - db.volume_attached(self.context, volume_id, instance_id, mountpoint) + db.volume_attached(self.context, volume_id, instance_id, + mountpoint) else: yield self.compute.attach_volume(self.context, instance_id, @@ -154,7 +156,8 @@ class VolumeTestCase(test.TrialTestCase): def _check(volume_id): """Make sure blades aren't duplicated""" volume_ids.append(volume_id) - (shelf_id, blade_id) = db.volume_get_shelf_and_blade(context.get_admin_context(), + admin_context = context.get_admin_context() + (shelf_id, blade_id) = db.volume_get_shelf_and_blade(admin_context, volume_id) shelf_blade = '%s.%s' % (shelf_id, blade_id) self.assert_(shelf_blade not in shelf_blades) -- cgit From 48fcb757852f6d408695a1b4467f59b4be22dbdd Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 22 Oct 2010 01:54:58 -0700 Subject: speed up disk generation by increasing block size --- nova/compute/disk.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/nova/compute/disk.py b/nova/compute/disk.py index 447a29004..1c9cfb330 100644 --- a/nova/compute/disk.py +++ b/nova/compute/disk.py @@ -34,6 +34,8 @@ from nova import flags FLAGS = flags.FLAGS flags.DEFINE_integer('minimum_root_size', 1024 * 1024 * 1024 * 10, 'minimum size in bytes of root partition') +flags.DEFINE_integer('block_size', 1024 * 1024 * 256, + 'block_size to use for dd') @defer.inlineCallbacks @@ -81,24 +83,27 @@ def partition(infile, outfile, local_bytes=0, resize=True, # create an empty file yield execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' - % (outfile, last_sector, sector_size)) + % (outfile, mbr_last, sector_size)) # make mbr partition yield execute('parted --script %s mklabel msdos' % outfile) + # append primary file + yield execute('dd if=%s of=%s bs=%s conv=notrunc,fsync oflag=append' + % (infile, outfile, FLAGS.block_size)) + # make primary partition yield execute('parted --script %s mkpart primary %ds %ds' % (outfile, primary_first, primary_last)) - # make local partition if local_bytes > 0: + # make the file bigger + yield execute('dd if=/dev/zero of=%s count=1 seek=%d bs=%d' + % (outfile, last_sector, sector_size)) + # make and format local partition yield execute('parted --script %s mkpartfs primary %s %ds %ds' % (outfile, local_type, local_first, local_last)) - # copy file into partition - yield execute('dd if=%s of=%s bs=%d seek=%d conv=notrunc,fsync' - % (infile, outfile, sector_size, primary_first)) - @defer.inlineCallbacks def inject_data(image, key=None, net=None, partition=None, execute=None): -- cgit From 0d10dcadc1186b023e7bcce7372c722c54ffe3a3 Mon Sep 17 00:00:00 2001 From: Rick Clark Date: Fri, 22 Oct 2010 13:47:22 -0500 Subject: Created Authors file and added to manifest for Austin Release --- MANIFEST.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MANIFEST.in b/MANIFEST.in index 562aa3f7b..4fe5f0b34 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,6 +1,6 @@ include HACKING LICENSE run_tests.py run_tests.sh include README builddeb.sh exercise_rsapi.py -include ChangeLog MANIFEST.in pylintrc +include ChangeLog MANIFEST.in pylintrc Authors graft CA graft doc graft smoketests -- cgit From 947203a467859fe9e0dea45a2a3590859611baed Mon Sep 17 00:00:00 2001 From: Rick Clark Date: Fri, 22 Oct 2010 14:06:34 -0500 Subject: Actually adding Authors file. --- Authors | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 Authors diff --git a/Authors b/Authors new file mode 100644 index 000000000..ec3a1cbd8 --- /dev/null +++ b/Authors @@ -0,0 +1,21 @@ +Andy Smith +Anne Gentle +Chris Behrens +Devin Carlen +Eric Day +Ewan Mellor +Hisaki Ohara +Jay Pipes +Jesse Andrews +Joe Heck +Joel Moore joelbm24@gmail.com +Joshua McKenty +Justin Santa Barbara +Matt Dietz +Michael Gundlach +Monty Taylor +Paul Voccio +Rick Clark +Soren Hansen +Todd Willey +Vishvananda Ishaya -- cgit From 5fdcbd6c831cb3ab2cb04c0eecc68e4b0b9d5a66 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 15:06:42 -0700 Subject: update tests --- nova/tests/virt_unittest.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/virt_unittest.py b/nova/tests/virt_unittest.py index ce78d450c..d49383fb7 100644 --- a/nova/tests/virt_unittest.py +++ b/nova/tests/virt_unittest.py @@ -91,7 +91,7 @@ class LibvirtConnTestCase(test.TrialTestCase): FLAGS.libvirt_type = libvirt_type conn = libvirt_conn.LibvirtConnection(True) - uri, template = conn.get_uri_and_template() + uri, _template, _rescue = conn.get_uri_and_templates() self.assertEquals(uri, expected_uri) xml = conn.to_xml(instance_ref) @@ -114,7 +114,7 @@ class LibvirtConnTestCase(test.TrialTestCase): for (libvirt_type, (expected_uri, checks)) in type_uri_map.iteritems(): FLAGS.libvirt_type = libvirt_type conn = libvirt_conn.LibvirtConnection(True) - uri, template = conn.get_uri_and_template() + uri, _template, _rescue = conn.get_uri_and_templates() self.assertEquals(uri, testuri) def tearDown(self): -- cgit From eecef70fcdd173cc54ad8ac7edba9e9b31855134 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 15:37:55 -0700 Subject: add methods to cloud for rescue and unrescue --- nova/api/cloud.py | 18 ++++++++++++++++++ nova/api/ec2/cloud.py | 17 +++++++++++++++-- nova/virt/libvirt_conn.py | 2 +- 3 files changed, 34 insertions(+), 3 deletions(-) diff --git a/nova/api/cloud.py b/nova/api/cloud.py index aa84075dc..46b342d32 100644 --- a/nova/api/cloud.py +++ b/nova/api/cloud.py @@ -36,3 +36,21 @@ def reboot(instance_id, context=None): db.queue_get_for(context, FLAGS.compute_topic, host), {"method": "reboot_instance", "args": {"instance_id": instance_ref['id']}}) + +def rescue(instance_id, context): + """Rescue the given instance.""" + instance_ref = db.instance_get_by_internal_id(context, instance_id) + host = instance_ref['host'] + rpc.cast(context, + db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "rescue_instance", + "args": {"instance_id": instance_ref['id']}}) + +def unrescue(instance_id, context): + """Unrescue the given instance.""" + instance_ref = db.instance_get_by_internal_id(context, instance_id) + host = instance_ref['host'] + rpc.cast(context, + db.queue_get_for(context, FLAGS.compute_topic, host), + {"method": "unrescue_instance", + "args": {"instance_id": instance_ref['id']}}) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 784697b01..73f0dcc16 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -932,8 +932,21 @@ class CloudController(object): def reboot_instances(self, context, instance_id, **kwargs): """instance_id is a list of instance ids""" - for id_str in instance_id: - cloud.reboot(id_str, context=context) + for ec2_id in instance_id: + internal_id = ec2_id_to_internal_id(ec2_id) + cloud.reboot(internal_id, context=context) + return True + + def rescue_instance(self, context, instance_id, **kwargs): + """This is an extension to the normal ec2_api""" + internal_id = ec2_id_to_internal_id(instance_id) + cloud.rescue(internal_id, context=context) + return True + + def unrescue_instance(self, context, instance_id, **kwargs): + """This is an extension to the normal ec2_api""" + internal_id = ec2_id_to_internal_id(instance_id) + cloud.unrescue(internal_id, context=context) return True def update_instance(self, context, ec2_id, **kwargs): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 1c75150ea..7d66d8454 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -484,7 +484,7 @@ class LibvirtConnection(object): try: virt_dom = self._conn.lookupByName(instance_name) except: - raise NotFound("Instance %s not found" % instance_name) + raise exception.NotFound("Instance %s not found" % instance_name) (state, max_mem, mem, num_cpu, cpu_time) = virt_dom.info() return {'state': state, 'max_mem': max_mem, -- cgit From 4948fed24d5e16d95f237ec95c1cd305fcc4df95 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 16:04:35 -0700 Subject: pep 8 cleanup and typo in resize --- nova/api/cloud.py | 2 ++ nova/virt/libvirt_conn.py | 6 ++++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/nova/api/cloud.py b/nova/api/cloud.py index 46b342d32..b8f15019f 100644 --- a/nova/api/cloud.py +++ b/nova/api/cloud.py @@ -37,6 +37,7 @@ def reboot(instance_id, context=None): {"method": "reboot_instance", "args": {"instance_id": instance_ref['id']}}) + def rescue(instance_id, context): """Rescue the given instance.""" instance_ref = db.instance_get_by_internal_id(context, instance_id) @@ -46,6 +47,7 @@ def rescue(instance_id, context): {"method": "rescue_instance", "args": {"instance_id": instance_ref['id']}}) + def unrescue(instance_id, context): """Unrescue the given instance.""" instance_ref = db.instance_get_by_internal_id(context, instance_id) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 7d66d8454..0096b1400 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -99,7 +99,7 @@ class LibvirtConnection(object): def __init__(self, read_only): (self.libvirt_uri, template_file, - rescue_file)= self.get_uri_and_templates() + rescue_file) = self.get_uri_and_templates() self.libvirt_xml = open(template_file).read() self.rescue_xml = open(rescue_file).read() @@ -258,6 +258,7 @@ class LibvirtConnection(object): d = defer.Deferred() timer = task.LoopingCall(f=None) + def _wait_for_rescue(): try: state = self.get_info(instance['name'])['state'] @@ -273,6 +274,7 @@ class LibvirtConnection(object): power_state.SHUTDOWN) timer.stop() d.callback(None) + timer.f = _wait_for_rescue timer.start(interval=0.5, now=True) yield d @@ -441,7 +443,7 @@ class LibvirtConnection(object): * 1024 * 1024 * 1024) resize = True - if inst['instance_type'] == 'm1.tiny' or prefix=='rescue': + if inst['instance_type'] == 'm1.tiny' or prefix == 'rescue-': resize = False yield disk.partition(basepath('disk-raw'), basepath('disk'), local_bytes, resize, execute=execute) -- cgit From a80b06285d993696ccb90eff00bb2963df49ecc6 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 17:18:24 -0700 Subject: add in the xen rescue template --- nova/virt/libvirt.rescue.xen.xml.template | 34 +++++++++++++++++++++++++++++++ nova/virt/libvirt.xen.xml.template | 6 +----- nova/virt/libvirt_conn.py | 3 +++ 3 files changed, 38 insertions(+), 5 deletions(-) create mode 100644 nova/virt/libvirt.rescue.xen.xml.template diff --git a/nova/virt/libvirt.rescue.xen.xml.template b/nova/virt/libvirt.rescue.xen.xml.template new file mode 100644 index 000000000..3b8d27237 --- /dev/null +++ b/nova/virt/libvirt.rescue.xen.xml.template @@ -0,0 +1,34 @@ + + %(name)s + + linux + %(basepath)s/kernel + %(basepath)s/ramdisk + /dev/xvda1 + ro + + + + + %(memory_kb)s + %(vcpus)s + + + + + + + + + + + + + + + + + + + + diff --git a/nova/virt/libvirt.xen.xml.template b/nova/virt/libvirt.xen.xml.template index 3b8d27237..9677902c6 100644 --- a/nova/virt/libvirt.xen.xml.template +++ b/nova/virt/libvirt.xen.xml.template @@ -13,13 +13,9 @@ %(memory_kb)s %(vcpus)s - - - - - + diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 0096b1400..e32945fa5 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -51,6 +51,9 @@ FLAGS = flags.FLAGS flags.DEFINE_string('libvirt_rescue_xml_template', utils.abspath('virt/libvirt.rescue.qemu.xml.template'), 'Libvirt RESCUE XML Template for QEmu/KVM') +flags.DEFINE_string('libvirt_rescue_xen_xml_template', + utils.abspath('virt/libvirt.rescue.xen.xml.template'), + 'Libvirt RESCUE XML Template for xen') flags.DEFINE_string('libvirt_rescue_uml_xml_template', utils.abspath('virt/libvirt.rescue.uml.xml.template'), 'Libvirt RESCUE XML Template for user-mode-linux') -- cgit From 9ee74816c0c2a28f7d056d524111cd940513766a Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 17:22:29 -0700 Subject: add NotFound to fake.py and document it --- nova/virt/fake.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nova/virt/fake.py b/nova/virt/fake.py index dae2a2410..66eff4c66 100644 --- a/nova/virt/fake.py +++ b/nova/virt/fake.py @@ -22,10 +22,9 @@ A fake (in-memory) hypervisor+api. Allows nova testing w/o a hypervisor. This module also documents the semantics of real hypervisor connections. """ -import logging - from twisted.internet import defer +from nova import exception from nova.compute import power_state @@ -160,7 +159,12 @@ class FakeConnection(object): current memory the instance has, in KiB, 'num_cpu': The current number of virtual CPUs the instance has, 'cpu_time': The total CPU time used by the instance, in nanoseconds. + + This method should raise exception.NotFound if the hypervisor has no + knowledge of the instance """ + if instance_name not in self.instances: + raise exception.NotFound("Instance %s Not Found" % instance_name) i = self.instances[instance_name] return {'state': i._state, 'max_mem': 0, -- cgit From 0c7b1ea7972defe67d8bebf4f23d189cc7b0422c Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 19:52:02 -0700 Subject: logging.warn not raise logging.Warn --- nova/compute/manager.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nova/compute/manager.py b/nova/compute/manager.py index fb9a4cb39..574feec7c 100644 --- a/nova/compute/manager.py +++ b/nova/compute/manager.py @@ -132,11 +132,11 @@ class ComputeManager(manager.Manager): self._update_state(context, instance_id) if instance_ref['state'] != power_state.RUNNING: - raise logging.Warn('trying to reboot a non-running ' - 'instance: %s (state: %s excepted: %s)', - instance_ref['internal_id'], - instance_ref['state'], - power_state.RUNNING) + logging.warn('trying to reboot a non-running ' + 'instance: %s (state: %s excepted: %s)', + instance_ref['internal_id'], + instance_ref['state'], + power_state.RUNNING) logging.debug('instance %s: rebooting', instance_ref['name']) self.db.instance_set_state(context, -- cgit From 0e98d027d1deb8cd46ddb9a1df4558a5c8b2edfc Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Sun, 24 Oct 2010 23:26:03 -0700 Subject: Removed unused imports and left over references to str_id --- nova/db/sqlalchemy/models.py | 25 +------------------------ nova/network/manager.py | 2 +- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/nova/db/sqlalchemy/models.py b/nova/db/sqlalchemy/models.py index 38c96bdec..2a3cfa94c 100644 --- a/nova/db/sqlalchemy/models.py +++ b/nova/db/sqlalchemy/models.py @@ -20,11 +20,9 @@ SQLAlchemy models for nova data """ -import sys import datetime -# TODO(vish): clean up these imports -from sqlalchemy.orm import relationship, backref, exc, object_mapper +from sqlalchemy.orm import relationship, backref, object_mapper from sqlalchemy import Column, Integer, String, schema from sqlalchemy import ForeignKey, DateTime, Boolean, Text from sqlalchemy.exc import IntegrityError @@ -46,17 +44,11 @@ class NovaBase(object): """Base class for Nova Models""" __table_args__ = {'mysql_engine': 'InnoDB'} __table_initialized__ = False - __prefix__ = 'none' created_at = Column(DateTime, default=datetime.datetime.utcnow) updated_at = Column(DateTime, onupdate=datetime.datetime.utcnow) deleted_at = Column(DateTime) deleted = Column(Boolean, default=False) - @property - def str_id(self): - """Get string id of object (generally prefix + '-' + id)""" - return "%s-%s" % (self.__prefix__, self.id) - def save(self, session=None): """Save this object""" if not session: @@ -94,7 +86,6 @@ class NovaBase(object): #class Image(BASE, NovaBase): # """Represents an image in the datastore""" # __tablename__ = 'images' -# __prefix__ = 'ami' # id = Column(Integer, primary_key=True) # ec2_id = Column(String(12), unique=True) # user_id = Column(String(255)) @@ -150,7 +141,6 @@ class Service(BASE, NovaBase): class Instance(BASE, NovaBase): """Represents a guest vm""" __tablename__ = 'instances' - __prefix__ = 'i' id = Column(Integer, primary_key=True) internal_id = Column(Integer, unique=True) @@ -227,7 +217,6 @@ class Instance(BASE, NovaBase): class Volume(BASE, NovaBase): """Represents a block storage device that can be attached to a vm""" __tablename__ = 'volumes' - __prefix__ = 'vol' id = Column(Integer, primary_key=True) ec2_id = Column(String(12), unique=True) @@ -269,10 +258,6 @@ class Quota(BASE, NovaBase): gigabytes = Column(Integer) floating_ips = Column(Integer) - @property - def str_id(self): - return self.project_id - class ExportDevice(BASE, NovaBase): """Represates a shelf and blade that a volume can be exported on""" @@ -361,10 +346,6 @@ class KeyPair(BASE, NovaBase): fingerprint = Column(String(255)) public_key = Column(Text) - @property - def str_id(self): - return '%s.%s' % (self.user_id, self.name) - class Network(BASE, NovaBase): """Represents a network""" @@ -426,10 +407,6 @@ class FixedIp(BASE, NovaBase): leased = Column(Boolean, default=False) reserved = Column(Boolean, default=False) - @property - def str_id(self): - return self.address - class User(BASE, NovaBase): """Represents a user""" diff --git a/nova/network/manager.py b/nova/network/manager.py index fddb77663..8a20cb491 100644 --- a/nova/network/manager.py +++ b/nova/network/manager.py @@ -171,7 +171,7 @@ class NetworkManager(manager.Manager): if not fixed_ip_ref['leased']: logging.warn("IP %s released that was not leased", address) self.db.fixed_ip_update(context, - fixed_ip_ref['str_id'], + fixed_ip_ref['address'], {'leased': False}) if not fixed_ip_ref['allocated']: self.db.fixed_ip_disassociate(context, address) -- cgit From 2e67031ffb981ae1a47043cd48d50470eb6dc0b6 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Mon, 25 Oct 2010 19:21:09 +0900 Subject: Duplicate the two trivial escaping functions remaining from tornado's code and remove the dependency. --- nova/objectstore/handler.py | 12 ++++++------ nova/utils.py | 25 +++++++++++++++++++++++++ tools/pip-requires | 1 - 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/nova/objectstore/handler.py b/nova/objectstore/handler.py index 074eea601..b26906001 100644 --- a/nova/objectstore/handler.py +++ b/nova/objectstore/handler.py @@ -44,7 +44,6 @@ import multiprocessing import os import urllib -from tornado import escape from twisted.application import internet from twisted.application import service from twisted.web import error @@ -55,6 +54,7 @@ from twisted.web import static from nova import context from nova import exception from nova import flags +from nova import utils from nova.auth import manager from nova.objectstore import bucket from nova.objectstore import image @@ -70,10 +70,10 @@ def render_xml(request, value): name = value.keys()[0] request.write('\n') - request.write('<' + escape.utf8(name) + + request.write('<' + utils.utf8(name) + ' xmlns="http://doc.s3.amazonaws.com/2006-03-01">') _render_parts(value.values()[0], request.write) - request.write('') + request.write('') request.finish() @@ -87,7 +87,7 @@ def finish(request, content=None): def _render_parts(value, write_cb): """Helper method to render different Python objects to XML""" if isinstance(value, basestring): - write_cb(escape.xhtml_escape(value)) + write_cb(utils.xhtml_escape(value)) elif isinstance(value, int) or isinstance(value, long): write_cb(str(value)) elif isinstance(value, datetime.datetime): @@ -97,9 +97,9 @@ def _render_parts(value, write_cb): if not isinstance(subvalue, list): subvalue = [subvalue] for subsubvalue in subvalue: - write_cb('<' + escape.utf8(name) + '>') + write_cb('<' + utils.utf8(name) + '>') _render_parts(subsubvalue, write_cb) - write_cb('') + write_cb('') else: raise Exception("Unknown S3 value type %r", value) diff --git a/nova/utils.py b/nova/utils.py index 7683fc9f4..e58302c11 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -28,6 +28,7 @@ import random import subprocess import socket import sys +from xml.sax import saxutils from twisted.internet.threads import deferToThread @@ -212,3 +213,27 @@ def deferredToThread(f): def g(*args, **kwargs): return deferToThread(f, *args, **kwargs) return g + + +def xhtml_escape(value): + """Escapes a string so it is valid within XML or XHTML. + + Code is directly from the utf8 function in + http://github.com/facebook/tornado/blob/master/tornado/escape.py + + """ + return saxutils.escape(value, {'"': """}) + + +def utf8(value): + """Try to turn a string into utf-8 if possible. + + Code is directly from the utf8 function in + http://github.com/facebook/tornado/blob/master/tornado/escape.py + + """ + if isinstance(value, unicode): + return value.encode("utf-8") + assert isinstance(value, str) + return value + diff --git a/tools/pip-requires b/tools/pip-requires index c76fad86f..548073326 100644 --- a/tools/pip-requires +++ b/tools/pip-requires @@ -13,7 +13,6 @@ python-daemon==1.5.5 python-gflags==1.3 redis==2.0.0 routes==1.12.3 -tornado==1.0 WebOb==0.9.8 wsgiref==0.1.2 zope.interface==3.6.1 -- cgit From e2746b3e4a3e07b6eea6c3db1551d4d61f2e0a97 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 25 Oct 2010 12:23:55 -0400 Subject: pep8 --- nova/api/openstack/ratelimiting/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/api/openstack/ratelimiting/__init__.py b/nova/api/openstack/ratelimiting/__init__.py index 9e028ecf5..918caf055 100644 --- a/nova/api/openstack/ratelimiting/__init__.py +++ b/nova/api/openstack/ratelimiting/__init__.py @@ -68,10 +68,10 @@ class Limiter(object): self._levels[key] = (now, new_level) return None - # If one instance of this WSGIApps is unable to handle your load, put a # sharding app in front that shards by username to one of many backends. + class WSGIApp(object): """Application that tracks rate limits in memory. Send requests to it of -- cgit From daa2569eda7a744113813e2fd4747c2f3e05e0c1 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 25 Oct 2010 12:25:50 -0400 Subject: pep8 --- nova/tests/api/test_wsgi.py | 2 +- nova/tests/api_unittest.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index 604d0cb66..44e2d615c 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -112,7 +112,7 @@ class SerializerTest(unittest.TestCase): self.match('/servers/4.json', None, expect='json') self.match('/servers/4', 'application/json', expect='json') self.match('/servers/4', 'application/xml', expect='xml') - self.match('/servers/4.xml', None, expect='xml') + self.match('/servers/4.xml', None, expect='xml') def test_defaults_to_json(self): self.match('/servers/4', None, expect='json') diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 80493b10a..0b1c3e353 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -242,7 +242,7 @@ class ApiEc2TestCase(test.BaseTestCase): self.assertEquals(int(group.rules[0].from_port), 80) self.assertEquals(int(group.rules[0].to_port), 81) self.assertEquals(len(group.rules[0].grants), 1) - self.assertEquals(str(group.rules[0].grants[0]), '0.0.0.0/0') + self.assertEquals(str(group.rules[0].grants[0]), '0.0.0.0/0') self.expect_http() self.mox.ReplayAll() -- cgit From 02e0b75e85f753043fc71ac2e0714ec4d4b0cca8 Mon Sep 17 00:00:00 2001 From: Jesse Andrews Date: Mon, 25 Oct 2010 10:43:03 -0700 Subject: update error message --- nova/api/ec2/cloud.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index bd7fbc3ac..62a8c475c 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -535,7 +535,8 @@ class CloudController(object): def attach_volume(self, context, volume_id, instance_id, device, **kwargs): volume_ref = db.volume_get_by_ec2_id(context, volume_id) if not re.match("^/dev/[a-z]d[a-z]+$", device): - raise exception.ApiError("Invalid device. Example /dev/vdb") + raise exception.ApiError("Invalid device specified: %s. " + "Example device: /dev/vdb" % device) # TODO(vish): abstract status checking? if volume_ref['status'] != "available": raise exception.ApiError("Volume status must be available") -- cgit From 5318bf110019d820e6f000662194d6e29f3e315f Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 25 Oct 2010 17:15:56 -0700 Subject: fix tests by removing missed reference to prefix and unnecessary conditional in generate_uid --- nova/db/sqlalchemy/api.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py index 0cbe56499..a3d8dde2f 100644 --- a/nova/db/sqlalchemy/api.py +++ b/nova/db/sqlalchemy/api.py @@ -19,6 +19,7 @@ Implementation of SQLAlchemy backend """ +import random import warnings from nova import db @@ -542,7 +543,8 @@ def instance_create(context, values): session = get_session() with session.begin(): while instance_ref.internal_id == None: - internal_id = utils.generate_uid(instance_ref.__prefix__) + # Instances have integer internal ids. + internal_id = random.randint(0, 2 ** 32 - 1) if not instance_internal_id_exists(context, internal_id, session=session): instance_ref.internal_id = internal_id @@ -1152,7 +1154,7 @@ def volume_create(context, values): session = get_session() with session.begin(): while volume_ref.ec2_id == None: - ec2_id = utils.generate_uid(volume_ref.__prefix__) + ec2_id = utils.generate_uid('vol') if not volume_ec2_id_exists(context, ec2_id, session=session): volume_ref.ec2_id = ec2_id volume_ref.save(session=session) -- cgit From 8ccdae97558d9660a9a0fac8dad809a09cbd3c71 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 25 Oct 2010 17:20:10 -0700 Subject: actually remove the conditional --- nova/utils.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 7683fc9f4..7f6311209 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -131,13 +131,9 @@ def runthis(prompt, cmd, check_exit_code=True): def generate_uid(topic, size=8): - if topic == "i": - # Instances have integer internal ids. - return random.randint(0, 2 ** 32 - 1) - else: - characters = '01234567890abcdefghijklmnopqrstuvwxyz' - choices = [random.choice(characters) for x in xrange(size)] - return '%s-%s' % (topic, ''.join(choices)) + characters = '01234567890abcdefghijklmnopqrstuvwxyz' + choices = [random.choice(characters) for x in xrange(size)] + return '%s-%s' % (topic, ''.join(choices)) def generate_mac(): -- cgit From 627a968e79ed21d970225e5ece332d9100abe022 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Mon, 25 Oct 2010 23:04:49 -0700 Subject: fix completely broken ServiceTestCase --- nova/tests/service_unittest.py | 125 +++++++++++++++++++++++------------------ 1 file changed, 70 insertions(+), 55 deletions(-) diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index e74e0f726..a268bc4fe 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -23,8 +23,8 @@ Unit Tests for remote procedure calls using queue import mox from twisted.application.app import startApplication +from twisted.internet import defer -from nova import context from nova import exception from nova import flags from nova import rpc @@ -48,7 +48,7 @@ class ExtendedService(service.Service): return 'service' -class ServiceManagerTestCase(test.BaseTestCase): +class ServiceManagerTestCase(test.TrialTestCase): """Test cases for Services""" def test_attribute_error_for_no_manager(self): @@ -75,13 +75,12 @@ class ServiceManagerTestCase(test.BaseTestCase): self.assertEqual(serv.test_method(), 'service') -class ServiceTestCase(test.BaseTestCase): +class ServiceTestCase(test.TrialTestCase): """Test cases for Services""" def setUp(self): super(ServiceTestCase, self).setUp() self.mox.StubOutWithMock(service, 'db') - self.context = context.get_admin_context() def test_create(self): host = 'foo' @@ -144,87 +143,103 @@ class ServiceTestCase(test.BaseTestCase): # whether it is disconnected, it looks for a variable on itself called # 'model_disconnected' and report_state doesn't really do much so this # these are mostly just for coverage - def test_report_state(self): - host = 'foo' - binary = 'bar' - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) - - self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - + @defer.inlineCallbacks def test_report_state_no_service(self): host = 'foo' binary = 'bar' + topic = 'test' service_create = {'host': host, 'binary': binary, + 'topic': topic, 'report_count': 0} service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} + 'binary': binary, + 'topic': topic, + 'report_count': 0, + 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, + service.db.service_get_by_args(mox.IgnoreArg(), host, binary).AndRaise(exception.NotFound()) - service.db.service_create(self.context, + service.db.service_create(mox.IgnoreArg(), service_create).AndReturn(service_ref) - service.db.service_get(self.context, + service.db.service_get(mox.IgnoreArg(), service_ref['id']).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], + service.db.service_update(mox.IgnoreArg(), service_ref['id'], mox.ContainsKeyValue('report_count', 1)) self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) + serv = service.Service(host, + binary, + topic, + 'nova.tests.service_unittest.FakeManager') + serv.startService() + yield serv.report_state() + @defer.inlineCallbacks def test_report_state_newly_disconnected(self): host = 'foo' binary = 'bar' + topic = 'test' + service_create = {'host': host, + 'binary': binary, + 'topic': topic, + 'report_count': 0} service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} + 'binary': binary, + 'topic': topic, + 'report_count': 0, + 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndRaise(Exception()) + service.db.service_get_by_args(mox.IgnoreArg(), + host, + binary).AndRaise(exception.NotFound()) + service.db.service_create(mox.IgnoreArg(), + service_create).AndReturn(service_ref) + service.db.service_get(mox.IgnoreArg(), + mox.IgnoreArg()).AndRaise(Exception()) self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - - self.assert_(s.model_disconnected) + serv = service.Service(host, + binary, + topic, + 'nova.tests.service_unittest.FakeManager') + serv.startService() + yield serv.report_state() + self.assert_(serv.model_disconnected) + @defer.inlineCallbacks def test_report_state_newly_connected(self): host = 'foo' binary = 'bar' + topic = 'test' + service_create = {'host': host, + 'binary': binary, + 'topic': topic, + 'report_count': 0} service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} + 'binary': binary, + 'topic': topic, + 'report_count': 0, + 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], + service.db.service_get_by_args(mox.IgnoreArg(), + host, + binary).AndRaise(exception.NotFound()) + service.db.service_create(mox.IgnoreArg(), + service_create).AndReturn(service_ref) + service.db.service_get(mox.IgnoreArg(), + service_ref['id']).AndReturn(service_ref) + service.db.service_update(mox.IgnoreArg(), service_ref['id'], mox.ContainsKeyValue('report_count', 1)) self.mox.ReplayAll() - s = service.Service() - s.model_disconnected = True - rv = yield s.report_state(host, binary) + serv = service.Service(host, + binary, + topic, + 'nova.tests.service_unittest.FakeManager') + serv.startService() + serv.model_disconnected = True + yield serv.report_state() - self.assert_(not s.model_disconnected) + self.assert_(not serv.model_disconnected) -- cgit From ba6d9293204284a7c74b5b0cffe63767941fd25c Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 26 Oct 2010 11:48:20 -0400 Subject: Delete BaseTestCase and with it the last reference to tornado. Requires commenting out some service_unittest tests which were silently failing under BaseTestCase and which now fail under TrialTestCase. vishy says he wrote the code and thinks he knows what is going wrong. --- nova/test.py | 156 ---------------------------------- nova/tests/api_unittest.py | 4 +- nova/tests/service_unittest.py | 184 +++++++++++++++++++++-------------------- run_tests.py | 16 ---- 4 files changed, 96 insertions(+), 264 deletions(-) diff --git a/nova/test.py b/nova/test.py index 8ef7eca1a..5c2a72819 100644 --- a/nova/test.py +++ b/nova/test.py @@ -28,7 +28,6 @@ import time import mox import stubout -from tornado import ioloop from twisted.internet import defer from twisted.trial import unittest @@ -159,158 +158,3 @@ class TrialTestCase(unittest.TestCase): _wrapped.func_name = self.originalAttach.func_name rpc.Consumer.attach_to_twisted = _wrapped - - -class BaseTestCase(TrialTestCase): - # TODO(jaypipes): Can this be moved into the TrialTestCase class? - """Base test case class for all unit tests. - - DEPRECATED: This is being removed once Tornado is gone, use TrialTestCase. - """ - def setUp(self): - """Run before each test method to initialize test environment""" - super(BaseTestCase, self).setUp() - # TODO(termie): we could possibly keep a more global registry of - # the injected listeners... this is fine for now though - self.ioloop = ioloop.IOLoop.instance() - - self._waiting = None - self._done_waiting = False - self._timed_out = False - - def _wait_for_test(self, timeout=60): - """ Push the ioloop along to wait for our test to complete. """ - self._waiting = self.ioloop.add_timeout(time.time() + timeout, - self._timeout) - - def _wait(): - - """Wrapped wait function. Called on timeout.""" - if self._timed_out: - self.fail('test timed out') - self._done() - if self._done_waiting: - self.ioloop.stop() - return - # we can use add_callback here but this uses less cpu when testing - self.ioloop.add_timeout(time.time() + 0.01, _wait) - - self.ioloop.add_callback(_wait) - self.ioloop.start() - - def _done(self): - """Callback used for cleaning up deferred test methods.""" - if self._waiting: - try: - self.ioloop.remove_timeout(self._waiting) - except Exception: # pylint: disable-msg=W0703 - # TODO(jaypipes): This produces a pylint warning. Should - # we really be catching Exception and then passing here? - pass - self._waiting = None - self._done_waiting = True - - def _maybe_inline_callbacks(self, func): - """ If we're doing async calls in our tests, wait on them. - - This is probably the most complicated hunk of code we have so far. - - First up, if the function is normal (not async) we just act normal - and return. - - Async tests will use the "Inline Callbacks" pattern, which means - you yield Deferreds at every "waiting" step of your code instead - of making epic callback chains. - - Example (callback chain, ugly): - - # A deferred instance - d = self.compute.terminate_instance(instance_id) - def _describe(_): - # Another deferred instance - d_desc = self.compute.describe_instances() - return d_desc - def _checkDescribe(rv): - self.assertEqual(rv, []) - d.addCallback(_describe) - d.addCallback(_checkDescribe) - d.addCallback(lambda x: self._done()) - self._wait_for_test() - - Example (inline callbacks! yay!): - - yield self.compute.terminate_instance(instance_id) - rv = yield self.compute.describe_instances() - self.assertEqual(rv, []) - - If the test fits the Inline Callbacks pattern we will automatically - handle calling wait and done. - """ - # TODO(termie): this can be a wrapper function instead and - # and we can make a metaclass so that we don't - # have to copy all that "run" code below. - g = func() - if not hasattr(g, 'send'): - self._done() - return defer.succeed(g) - - inlined = defer.inlineCallbacks(func) - d = inlined() - return d - - def _catch_exceptions(self, result, failure): - """Catches all exceptions and handles keyboard interrupts.""" - exc = (failure.type, failure.value, failure.getTracebackObject()) - if isinstance(failure.value, self.failureException): - result.addFailure(self, exc) - elif isinstance(failure.value, KeyboardInterrupt): - raise - else: - result.addError(self, exc) - - self._done() - - def _timeout(self): - """Helper method which trips the timeouts""" - self._waiting = False - self._timed_out = True - - def run(self, result=None): - """Runs the test case""" - - result.startTest(self) - test_method = getattr(self, self._testMethodName) - try: - try: - self.setUp() - except KeyboardInterrupt: - raise - except: - result.addError(self, sys.exc_info()) - return - - ok = False - try: - d = self._maybe_inline_callbacks(test_method) - d.addErrback(lambda x: self._catch_exceptions(result, x)) - d.addBoth(lambda x: self._done() and x) - self._wait_for_test() - ok = True - except self.failureException: - result.addFailure(self, sys.exc_info()) - except KeyboardInterrupt: - raise - except: - result.addError(self, sys.exc_info()) - - try: - self.tearDown() - except KeyboardInterrupt: - raise - except: - result.addError(self, sys.exc_info()) - ok = False - if ok: - result.addSuccess(self) - finally: - result.stopTest(self) diff --git a/nova/tests/api_unittest.py b/nova/tests/api_unittest.py index 0b1c3e353..0a81c575b 100644 --- a/nova/tests/api_unittest.py +++ b/nova/tests/api_unittest.py @@ -83,7 +83,7 @@ class FakeHttplibConnection(object): pass -class XmlConversionTestCase(test.BaseTestCase): +class XmlConversionTestCase(test.TrialTestCase): """Unit test api xml conversion""" def test_number_conversion(self): conv = apirequest._try_convert @@ -100,7 +100,7 @@ class XmlConversionTestCase(test.BaseTestCase): self.assertEqual(conv('-0'), 0) -class ApiEc2TestCase(test.BaseTestCase): +class ApiEc2TestCase(test.TrialTestCase): """Unit test for the cloud controller on an EC2 API""" def setUp(self): super(ApiEc2TestCase, self).setUp() diff --git a/nova/tests/service_unittest.py b/nova/tests/service_unittest.py index e74e0f726..142c2ebea 100644 --- a/nova/tests/service_unittest.py +++ b/nova/tests/service_unittest.py @@ -48,7 +48,7 @@ class ExtendedService(service.Service): return 'service' -class ServiceManagerTestCase(test.BaseTestCase): +class ServiceManagerTestCase(test.TrialTestCase): """Test cases for Services""" def test_attribute_error_for_no_manager(self): @@ -75,7 +75,7 @@ class ServiceManagerTestCase(test.BaseTestCase): self.assertEqual(serv.test_method(), 'service') -class ServiceTestCase(test.BaseTestCase): +class ServiceTestCase(test.TrialTestCase): """Test cases for Services""" def setUp(self): @@ -140,91 +140,95 @@ class ServiceTestCase(test.BaseTestCase): startApplication(app, False) self.assert_(app) - # We're testing sort of weird behavior in how report_state decides - # whether it is disconnected, it looks for a variable on itself called - # 'model_disconnected' and report_state doesn't really do much so this - # these are mostly just for coverage - def test_report_state(self): - host = 'foo' - binary = 'bar' - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) - - self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - - def test_report_state_no_service(self): - host = 'foo' - binary = 'bar' - service_create = {'host': host, - 'binary': binary, - 'report_count': 0} - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndRaise(exception.NotFound()) - service.db.service_create(self.context, - service_create).AndReturn(service_ref) - service.db.service_get(self.context, - service_ref['id']).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) - - self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - - def test_report_state_newly_disconnected(self): - host = 'foo' - binary = 'bar' - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndRaise(Exception()) - - self.mox.ReplayAll() - s = service.Service() - rv = yield s.report_state(host, binary) - - self.assert_(s.model_disconnected) - - def test_report_state_newly_connected(self): - host = 'foo' - binary = 'bar' - service_ref = {'host': host, - 'binary': binary, - 'report_count': 0, - 'id': 1} - - service.db.__getattr__('report_state') - service.db.service_get_by_args(self.context, - host, - binary).AndReturn(service_ref) - service.db.service_update(self.context, service_ref['id'], - mox.ContainsKeyValue('report_count', 1)) - - self.mox.ReplayAll() - s = service.Service() - s.model_disconnected = True - rv = yield s.report_state(host, binary) - - self.assert_(not s.model_disconnected) +# TODO(gundlach): These tests were "passing" when this class inherited from +# BaseTestCase. In reality, they were failing, but BaseTestCase was +# swallowing the error. Now that we inherit from TrialTestCase, these tests +# are failing, and need to get fixed. +# # We're testing sort of weird behavior in how report_state decides +# # whether it is disconnected, it looks for a variable on itself called +# # 'model_disconnected' and report_state doesn't really do much so this +# # these are mostly just for coverage +# def test_report_state(self): +# host = 'foo' +# binary = 'bar' +# service_ref = {'host': host, +# 'binary': binary, +# 'report_count': 0, +# 'id': 1} +# service.db.__getattr__('report_state') +# service.db.service_get_by_args(self.context, +# host, +# binary).AndReturn(service_ref) +# service.db.service_update(self.context, service_ref['id'], +# mox.ContainsKeyValue('report_count', 1)) +# +# self.mox.ReplayAll() +# s = service.Service() +# rv = yield s.report_state(host, binary) +# +# def test_report_state_no_service(self): +# host = 'foo' +# binary = 'bar' +# service_create = {'host': host, +# 'binary': binary, +# 'report_count': 0} +# service_ref = {'host': host, +# 'binary': binary, +# 'report_count': 0, +# 'id': 1} +# +# service.db.__getattr__('report_state') +# service.db.service_get_by_args(self.context, +# host, +# binary).AndRaise(exception.NotFound()) +# service.db.service_create(self.context, +# service_create).AndReturn(service_ref) +# service.db.service_get(self.context, +# service_ref['id']).AndReturn(service_ref) +# service.db.service_update(self.context, service_ref['id'], +# mox.ContainsKeyValue('report_count', 1)) +# +# self.mox.ReplayAll() +# s = service.Service() +# rv = yield s.report_state(host, binary) +# +# def test_report_state_newly_disconnected(self): +# host = 'foo' +# binary = 'bar' +# service_ref = {'host': host, +# 'binary': binary, +# 'report_count': 0, +# 'id': 1} +# +# service.db.__getattr__('report_state') +# service.db.service_get_by_args(self.context, +# host, +# binary).AndRaise(Exception()) +# +# self.mox.ReplayAll() +# s = service.Service() +# rv = yield s.report_state(host, binary) +# +# self.assert_(s.model_disconnected) +# +# def test_report_state_newly_connected(self): +# host = 'foo' +# binary = 'bar' +# service_ref = {'host': host, +# 'binary': binary, +# 'report_count': 0, +# 'id': 1} +# +# service.db.__getattr__('report_state') +# service.db.service_get_by_args(self.context, +# host, +# binary).AndReturn(service_ref) +# service.db.service_update(self.context, service_ref['id'], +# mox.ContainsKeyValue('report_count', 1)) +# +# self.mox.ReplayAll() +# s = service.Service() +# s.model_disconnected = True +# rv = yield s.report_state(host, binary) +# +# self.assert_(not s.model_disconnected) diff --git a/run_tests.py b/run_tests.py index 9a2f40dc9..c16c63249 100644 --- a/run_tests.py +++ b/run_tests.py @@ -48,24 +48,8 @@ from twisted.scripts import trial as trial_script from nova import flags from nova import twistd -from nova.tests.access_unittest import * from nova.tests.auth_unittest import * -from nova.tests.api_unittest import * -from nova.tests.cloud_unittest import * -from nova.tests.compute_unittest import * -from nova.tests.flags_unittest import * -from nova.tests.network_unittest import * -from nova.tests.objectstore_unittest import * -from nova.tests.process_unittest import * -from nova.tests.quota_unittest import * -from nova.tests.rpc_unittest import * -from nova.tests.scheduler_unittest import * from nova.tests.service_unittest import * -from nova.tests.twistd_unittest import * -from nova.tests.validator_unittest import * -from nova.tests.virt_unittest import * -from nova.tests.volume_unittest import * -from nova.tests.virt_unittest import * FLAGS = flags.FLAGS -- cgit From f0d79d7d602a31fff03d8d934203128a2cd8940d Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 26 Oct 2010 11:58:46 -0400 Subject: Oops, didn't mean to check this one in. Ninja-patch --- run_tests.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/run_tests.py b/run_tests.py index c16c63249..9a2f40dc9 100644 --- a/run_tests.py +++ b/run_tests.py @@ -48,8 +48,24 @@ from twisted.scripts import trial as trial_script from nova import flags from nova import twistd +from nova.tests.access_unittest import * from nova.tests.auth_unittest import * +from nova.tests.api_unittest import * +from nova.tests.cloud_unittest import * +from nova.tests.compute_unittest import * +from nova.tests.flags_unittest import * +from nova.tests.network_unittest import * +from nova.tests.objectstore_unittest import * +from nova.tests.process_unittest import * +from nova.tests.quota_unittest import * +from nova.tests.rpc_unittest import * +from nova.tests.scheduler_unittest import * from nova.tests.service_unittest import * +from nova.tests.twistd_unittest import * +from nova.tests.validator_unittest import * +from nova.tests.virt_unittest import * +from nova.tests.volume_unittest import * +from nova.tests.virt_unittest import * FLAGS = flags.FLAGS -- cgit From cce61a2d29fac66cdbef74517bf1ab025df33d1f Mon Sep 17 00:00:00 2001 From: Anne Gentle Date: Tue, 26 Oct 2010 09:06:37 -0500 Subject: Added Google Analytics code --- doc/source/_templates/layout.html | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 doc/source/_templates/layout.html diff --git a/doc/source/_templates/layout.html b/doc/source/_templates/layout.html new file mode 100644 index 000000000..0b72a77ac --- /dev/null +++ b/doc/source/_templates/layout.html @@ -0,0 +1,17 @@ +{% extends "!layout.html" %} + +{% block footer %} +{{ super() }} + + +{% endblock %} + -- cgit From d8d12549a5e47c7c44f449f12d6b556e2c56483d Mon Sep 17 00:00:00 2001 From: Eric Day Date: Tue, 26 Oct 2010 15:37:32 -0700 Subject: More PEP8 fixes that were introduced in the last couple commits. --- nova/tests/api/openstack/test_api.py | 6 ++++++ nova/utils.py | 5 ++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/nova/tests/api/openstack/test_api.py b/nova/tests/api/openstack/test_api.py index a8c0ff9f8..dd83991b9 100644 --- a/nova/tests/api/openstack/test_api.py +++ b/nova/tests/api/openstack/test_api.py @@ -24,22 +24,28 @@ from nova.api.openstack import API from nova.api.openstack import faults from webob import Request + class APITest(unittest.TestCase): def test_exceptions_are_converted_to_faults(self): + @webob.dec.wsgify def succeed(req): return 'Succeeded' + @webob.dec.wsgify def raise_webob_exc(req): raise webob.exc.HTTPNotFound(explanation='Raised a webob.exc') + @webob.dec.wsgify def fail(req): raise Exception("Threw an exception") + @webob.dec.wsgify def raise_api_fault(req): exc = webob.exc.HTTPNotFound(explanation='Raised a webob.exc') return faults.Fault(exc) + api = API() api.application = succeed diff --git a/nova/utils.py b/nova/utils.py index 2c53b027e..bc495a691 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -213,10 +213,10 @@ def deferredToThread(f): def xhtml_escape(value): """Escapes a string so it is valid within XML or XHTML. - + Code is directly from the utf8 function in http://github.com/facebook/tornado/blob/master/tornado/escape.py - + """ return saxutils.escape(value, {'"': """}) @@ -232,4 +232,3 @@ def utf8(value): return value.encode("utf-8") assert isinstance(value, str) return value - -- cgit