From fb6bf337bc2fe702307842b57e33b9f5f9011147 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Sun, 15 Aug 2010 22:48:54 +0100 Subject: Rework virt.xenapi's concurrency model. There were many places where we were inadvertently blocking the reactor thread. The reworking puts all calls to XenAPI on background threads, so that they won't block the reactor thread. Long-lived operations (VM start, reboot, etc) are invoked asynchronously at the XenAPI level (Async.VM.start, etc). These return a XenAPI task. We relinquish the background thread at this point, so as not to hold threads in the pool for too long, and use reactor.callLater to poll the task. This combination of techniques means that we don't block the reactor thread at all, and at the same time we don't hold lots of threads waiting for long-running operations. There is a FIXME in here: get_info does not conform to these new rules. Changes are required in compute.service before we can make get_info non-blocking. --- nova/virt/xenapi.py | 178 +++++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 147 insertions(+), 31 deletions(-) (limited to 'nova') diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py index 9fe15644f..6b41061c1 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi.py @@ -16,15 +16,33 @@ """ A connection to XenServer or Xen Cloud Platform. + +The concurrency model for this class is as follows: + +All XenAPI calls are on a thread (using t.i.t.deferToThread, or the decorator +deferredToThread). They are remote calls, and so may hang for the usual +reasons. They should not be allowed to block the reactor thread. + +All long-running XenAPI calls (VM.start, VM.reboot, etc) are called async +(using XenAPI.VM.async_start etc). These return a task, which can then be +polled for completion. Polling is handled using reactor.callLater. + +This combination of techniques means that we don't block the reactor thread at +all, and at the same time we don't hold lots of threads waiting for +long-running operations. + +FIXME: get_info currently doesn't conform to these rules, and will block the +reactor thread if the VM.get_by_name_label or VM.get_record calls block. """ import logging import xmlrpclib from twisted.internet import defer +from twisted.internet import reactor from twisted.internet import task +from twisted.internet.threads import deferToThread -from nova import exception from nova import flags from nova import process from nova.auth.manager import AuthManager @@ -43,6 +61,9 @@ flags.DEFINE_string('xenapi_connection_username', flags.DEFINE_string('xenapi_connection_password', None, 'Password for connection to XenServer/Xen Cloud Platform. Used only if connection_type=xenapi.') +flags.DEFINE_float('xenapi_task_poll_interval', + 0.5, + 'The interval used for polling of remote tasks (Async.VM.start, etc). Used only if connection_type=xenapi.') def get_connection(_): @@ -61,6 +82,12 @@ def get_connection(_): return XenAPIConnection(url, username, password) +def deferredToThread(f): + def g(*args, **kwargs): + return deferToThread(f, *args, **kwargs) + return g + + class XenAPIConnection(object): def __init__(self, url, user, pw): @@ -72,9 +99,8 @@ class XenAPIConnection(object): for vm in self._conn.xenapi.VM.get_all()] @defer.inlineCallbacks - @exception.wrap_exception def spawn(self, instance): - vm = yield self.lookup(instance.name) + vm = yield self._lookup(instance.name) if vm is not None: raise Exception('Attempted to create non-unique name %s' % instance.name) @@ -93,22 +119,28 @@ class XenAPIConnection(object): user = AuthManager().get_user(instance.datamodel['user_id']) project = AuthManager().get_project(instance.datamodel['project_id']) - vdi_uuid = yield self.fetch_image( + vdi_uuid = yield self._fetch_image( instance.datamodel['image_id'], user, project, True) - kernel = yield self.fetch_image( + kernel = yield self._fetch_image( instance.datamodel['kernel_id'], user, project, False) - ramdisk = yield self.fetch_image( + ramdisk = yield self._fetch_image( instance.datamodel['ramdisk_id'], user, project, False) - vdi_ref = yield self._conn.xenapi.VDI.get_by_uuid(vdi_uuid) + vdi_ref = yield self._call_xenapi('VDI.get_by_uuid', vdi_uuid) - vm_ref = yield self.create_vm(instance, kernel, ramdisk) - yield self.create_vbd(vm_ref, vdi_ref, 0, True) + vm_ref = yield self._create_vm(instance, kernel, ramdisk) + yield self._create_vbd(vm_ref, vdi_ref, 0, True) if network_ref: yield self._create_vif(vm_ref, network_ref, mac_address) - yield self._conn.xenapi.VM.start(vm_ref, False, False) + logging.debug('Starting VM %s...', vm_ref) + yield self._call_xenapi('VM.start', vm_ref, False, False) + logging.info('Spawning VM %s created %s.', instance.name, vm_ref) - def create_vm(self, instance, kernel, ramdisk): + @defer.inlineCallbacks + def _create_vm(self, instance, kernel, ramdisk): + """Create a VM record. Returns a Deferred that gives the new + VM reference.""" + mem = str(long(instance.datamodel['memory_kb']) * 1024) vcpus = str(instance.datamodel['vcpus']) rec = { @@ -141,12 +173,16 @@ class XenAPIConnection(object): 'other_config': {}, } logging.debug('Created VM %s...', instance.name) - vm_ref = self._conn.xenapi.VM.create(rec) + vm_ref = yield self._call_xenapi('VM.create', rec) logging.debug('Created VM %s as %s.', instance.name, vm_ref) - return vm_ref + defer.returnValue(vm_ref) - def create_vbd(self, vm_ref, vdi_ref, userdevice, bootable): + @defer.inlineCallbacks + def _create_vbd(self, vm_ref, vdi_ref, userdevice, bootable): + """Create a VBD record. Returns a Deferred that gives the new + VBD reference.""" + vbd_rec = {} vbd_rec['VM'] = vm_ref vbd_rec['VDI'] = vdi_ref @@ -161,13 +197,17 @@ class XenAPIConnection(object): vbd_rec['qos_algorithm_params'] = {} vbd_rec['qos_supported_algorithms'] = [] logging.debug('Creating VBD for VM %s, VDI %s ... ', vm_ref, vdi_ref) - vbd_ref = self._conn.xenapi.VBD.create(vbd_rec) + vbd_ref = yield self._call_xenapi('VBD.create', vbd_rec) logging.debug('Created VBD %s for VM %s, VDI %s.', vbd_ref, vm_ref, vdi_ref) - return vbd_ref + defer.returnValue(vbd_ref) + @defer.inlineCallbacks def _create_vif(self, vm_ref, network_ref, mac_address): + """Create a VIF record. Returns a Deferred that gives the new + VIF reference.""" + vif_rec = {} vif_rec['device'] = '0' vif_rec['network']= network_ref @@ -179,27 +219,31 @@ class XenAPIConnection(object): vif_rec['qos_algorithm_params'] = {} logging.debug('Creating VIF for VM %s, network %s ... ', vm_ref, network_ref) - vif_ref = self._conn.xenapi.VIF.create(vif_rec) + vif_ref = yield self._call_xenapi('VIF.create', vif_rec) logging.debug('Created VIF %s for VM %s, network %s.', vif_ref, vm_ref, network_ref) - return vif_ref + defer.returnValue(vif_ref) + @defer.inlineCallbacks def _find_network_with_bridge(self, bridge): expr = 'field "bridge" = "%s"' % bridge - networks = self._conn.xenapi.network.get_all_records_where(expr) + networks = yield self._call_xenapi('network.get_all_records_where', + expr) if len(networks) == 1: - return networks.keys()[0] + defer.returnValue(networks.keys()[0]) elif len(networks) > 1: raise Exception('Found non-unique network for bridge %s' % bridge) else: raise Exception('Found no network for bridge %s' % bridge) - def fetch_image(self, image, user, project, use_sr): + @defer.inlineCallbacks + def _fetch_image(self, image, user, project, use_sr): """use_sr: True to put the image as a VDI in an SR, False to place it on dom0's filesystem. The former is for VM disks, the latter for - its kernel and ramdisk (if external kernels are being used).""" + its kernel and ramdisk (if external kernels are being used). + Returns a Deferred that gives the new VDI UUID.""" url = images.image_url(image) access = AuthManager().get_access_key(user, project) @@ -211,23 +255,31 @@ class XenAPIConnection(object): args['password'] = user.secret if use_sr: args['add_partition'] = 'true' - return self._call_plugin('objectstore', fn, args) + task = yield self._async_call_plugin('objectstore', fn, args) + uuid = yield self._wait_for_task(task) + defer.returnValue(uuid) + @defer.inlineCallbacks def reboot(self, instance): - vm = self.lookup(instance.name) + vm = yield self._lookup(instance.name) if vm is None: raise Exception('instance not present %s' % instance.name) - yield self._conn.xenapi.VM.clean_reboot(vm) + task = yield self._call_xenapi('Async.VM.clean_reboot', vm) + yield self._wait_for_task(task) + + @defer.inlineCallbacks def destroy(self, instance): - vm = self.lookup(instance.name) + vm = yield self._lookup(instance.name) if vm is None: raise Exception('instance not present %s' % instance.name) - yield self._conn.xenapi.VM.destroy(vm) + task = yield self._call_xenapi('Async.VM.destroy', vm) + yield self._wait_for_task(task) + def get_info(self, instance_id): - vm = self.lookup(instance_id) + vm = self._lookup_blocking(instance_id) if vm is None: raise Exception('instance not present %s' % instance_id) rec = self._conn.xenapi.VM.get_record(vm) @@ -237,7 +289,13 @@ class XenAPIConnection(object): 'num_cpu': rec['VCPUs_max'], 'cpu_time': 0} - def lookup(self, i): + + @deferredToThread + def _lookup(self, i): + return self._lookup_blocking(i) + + + def _lookup_blocking(self, i): vms = self._conn.xenapi.VM.get_by_name_label(i) n = len(vms) if n == 0: @@ -248,9 +306,55 @@ class XenAPIConnection(object): return vms[0] - def _call_plugin(self, plugin, fn, args): + def _wait_for_task(self, task): + """Return a Deferred that will give the result of the given task. + The task is polled until it completes.""" + d = defer.Deferred() + reactor.callLater(0, self._poll_task, task, d) + return d + + + @deferredToThread + def _poll_task(self, task, deferred): + """Poll the given XenAPI task, and fire the given Deferred if we + get a result.""" + try: + #logging.debug('Polling task %s...', task) + status = self._conn.xenapi.task.get_status(task) + if status == 'pending': + reactor.callLater(FLAGS.xenapi_task_poll_interval, + self._poll_task, task, deferred) + elif status == 'success': + result = self._conn.xenapi.task.get_result(task) + logging.info('Task %s status: success. %s', task, result) + deferred.callback(_parse_xmlrpc_value(result)) + else: + error_info = self._conn.xenapi.task.get_error_info(task) + logging.warn('Task %s status: %s. %s', task, status, + error_info) + deferred.errback(XenAPI.Failure(error_info)) + #logging.debug('Polling task %s done.', task) + except Exception, exn: + logging.warn(exn) + deferred.errback(exn) + + + @deferredToThread + def _call_xenapi(self, method, *args): + """Call the specified XenAPI method on a background thread. Returns + a Deferred for the result.""" + f = self._conn.xenapi + for m in method.split('.'): + f = f.__getattr__(m) + return f(*args) + + + @deferredToThread + def _async_call_plugin(self, plugin, fn, args): + """Call Async.host.call_plugin on a background thread. Returns a + Deferred with the task reference.""" return _unwrap_plugin_exceptions( - self._conn.xenapi.host.call_plugin, + self._conn.xenapi.Async.host.call_plugin, self._get_xenapi_host(), plugin, fn, args) @@ -286,3 +390,15 @@ def _unwrap_plugin_exceptions(func, *args, **kwargs): except xmlrpclib.ProtocolError, exn: logging.debug("Got exception: %s", exn) raise + + +def _parse_xmlrpc_value(val): + """Parse the given value as if it were an XML-RPC value. This is + sometimes used as the format for the task.result field.""" + if not val: + return val + x = xmlrpclib.loads( + '' + + val + + '') + return x[0][0] -- cgit From 31c08591793311606551bf0e6bfc14b155b491a6 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 17 Aug 2010 16:46:19 +0200 Subject: Use the argument handler specified by twistd, if any. --- nova/flags.py | 3 +++ nova/server.py | 6 +++++- nova/twistd.py | 12 +++++++++++- 3 files changed, 19 insertions(+), 2 deletions(-) (limited to 'nova') diff --git a/nova/flags.py b/nova/flags.py index e3feb252d..e0181102e 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -141,6 +141,7 @@ def _wrapper(func): return _wrapped +DEFINE = _wrapper(gflags.DEFINE) DEFINE_string = _wrapper(gflags.DEFINE_string) DEFINE_integer = _wrapper(gflags.DEFINE_integer) DEFINE_bool = _wrapper(gflags.DEFINE_bool) @@ -152,6 +153,8 @@ DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist) DEFINE_multistring = _wrapper(gflags.DEFINE_multistring) DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int) +ArgumentSerializer = gflags.ArgumentSerializer + def DECLARE(name, module_string, flag_values=FLAGS): if module_string not in sys.modules: diff --git a/nova/server.py b/nova/server.py index 96550f078..c6b60e090 100644 --- a/nova/server.py +++ b/nova/server.py @@ -44,6 +44,8 @@ flags.DEFINE_bool('use_syslog', True, 'output to syslog when daemonizing') flags.DEFINE_string('logfile', None, 'log file to output to') flags.DEFINE_string('pidfile', None, 'pid file to output to') flags.DEFINE_string('working_directory', './', 'working directory...') +flags.DEFINE_integer('uid', os.getuid(), 'uid under which to run') +flags.DEFINE_integer('gid', os.getgid(), 'gid under which to run') def stop(pidfile): @@ -135,6 +137,8 @@ def daemonize(args, name, main): threaded=False), stdin=stdin, stdout=stdout, - stderr=stderr + stderr=stderr, + uid=FLAGS.uid, + gid=FLAGS.gid ): main(args) diff --git a/nova/twistd.py b/nova/twistd.py index 8de322aa5..a72cc85e6 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -48,6 +48,13 @@ class TwistdServerOptions(ServerOptions): def parseArgs(self, *args): return +class FlagParser(object): + def __init__(self, parser): + self.parser = parser + + def Parse(self, s): + return self.parser(s) + def WrapTwistedOptions(wrapped): class TwistedOptionsToFlags(wrapped): @@ -79,7 +86,10 @@ def WrapTwistedOptions(wrapped): reflect.accumulateClassList(self.__class__, 'optParameters', twistd_params) for param in twistd_params: key = param[0].replace('-', '_') - flags.DEFINE_string(key, param[2], str(param[-1])) + if len(param) > 4: + flags.DEFINE(FlagParser(param[4]), key, param[2], str(param[3]), serializer=flags.ArgumentSerializer()) + else: + flags.DEFINE_string(key, param[2], str(param[3])) def _absorbHandlers(self): twistd_handlers = {} -- cgit From 200daa3e5d5571add6c2937cf847641d065e87b8 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Wed, 18 Aug 2010 00:05:06 +0200 Subject: Stylistic improvements. --- nova/flags.py | 2 -- nova/twistd.py | 6 +++++- 2 files changed, 5 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/flags.py b/nova/flags.py index e0181102e..6f9f906dd 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -153,8 +153,6 @@ DEFINE_spaceseplist = _wrapper(gflags.DEFINE_spaceseplist) DEFINE_multistring = _wrapper(gflags.DEFINE_multistring) DEFINE_multi_int = _wrapper(gflags.DEFINE_multi_int) -ArgumentSerializer = gflags.ArgumentSerializer - def DECLARE(name, module_string, flag_values=FLAGS): if module_string not in sys.modules: diff --git a/nova/twistd.py b/nova/twistd.py index a72cc85e6..9511c231c 100644 --- a/nova/twistd.py +++ b/nova/twistd.py @@ -21,6 +21,7 @@ Twisted daemon helpers, specifically to parse out gFlags from twisted flags, manage pid files and support syslogging. """ +import gflags import logging import os import signal @@ -48,6 +49,7 @@ class TwistdServerOptions(ServerOptions): def parseArgs(self, *args): return + class FlagParser(object): def __init__(self, parser): self.parser = parser @@ -87,7 +89,9 @@ def WrapTwistedOptions(wrapped): for param in twistd_params: key = param[0].replace('-', '_') if len(param) > 4: - flags.DEFINE(FlagParser(param[4]), key, param[2], str(param[3]), serializer=flags.ArgumentSerializer()) + flags.DEFINE(FlagParser(param[4]), + key, param[2], str(param[3]), + serializer=gflags.ArgumentSerializer()) else: flags.DEFINE_string(key, param[2], str(param[3])) -- cgit From 24a6fd40f657896fb20249392be6ed41c30ca679 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Wed, 18 Aug 2010 11:19:40 -0400 Subject: Image API work --- nova/endpoint/newapi.py | 4 --- nova/endpoint/rackspace/controllers/base.py | 9 +++++ nova/endpoint/rackspace/controllers/images.py | 48 ++++++++++++++++++++++++++- 3 files changed, 56 insertions(+), 5 deletions(-) (limited to 'nova') diff --git a/nova/endpoint/newapi.py b/nova/endpoint/newapi.py index 9aae933af..7836be582 100644 --- a/nova/endpoint/newapi.py +++ b/nova/endpoint/newapi.py @@ -41,11 +41,7 @@ class APIVersionRouter(wsgi.Router): def __init__(self): mapper = routes.Mapper() - rsapi = rackspace.API() mapper.connect(None, "/v1.0/{path_info:.*}", controller=rsapi) - mapper.connect(None, "/ec2/{path_info:.*}", controller=aws.API()) - super(APIVersionRouter, self).__init__(mapper) - diff --git a/nova/endpoint/rackspace/controllers/base.py b/nova/endpoint/rackspace/controllers/base.py index 8cd44f62e..88922280b 100644 --- a/nova/endpoint/rackspace/controllers/base.py +++ b/nova/endpoint/rackspace/controllers/base.py @@ -7,3 +7,12 @@ class BaseController(wsgi.Controller): return { cls.entity_name : cls.render(instance) } else: return { "TODO": "TODO" } + + def serialize(self, data, request): + """ + Serialize the given dict to the response type requested in request. + Uses self._serialization_metadata if it exists, which is a dict mapping + MIME types to information needed to serialize to that type. + """ + _metadata = getattr(type(self), "_serialization_metadata", {}) + return Serializer(request.environ, _metadata).to_content_type(data) diff --git a/nova/endpoint/rackspace/controllers/images.py b/nova/endpoint/rackspace/controllers/images.py index ae2a08849..197d8375c 100644 --- a/nova/endpoint/rackspace/controllers/images.py +++ b/nova/endpoint/rackspace/controllers/images.py @@ -1 +1,47 @@ -class ImagesController(object): pass +from nova.endpoint.rackspace.controllers.base import BaseController +from nova.endpoint import images +from webob import exc + +#TODO(gundlach): Serialize return values +class ImagesController(BaseController): + + _serialization_metadata = { + 'application/xml': { + "attributes": { + "image": [ "id", "name", "updated", "created", "status", + "serverId", "progress" ] + } + } + } + + def index(self, req): + context = req.environ['nova.api_request_context'] + return images.list(context) + + def show(self, req, id): + context = req.environ['nova.api_request_context'] + return images.list(context, filter_list=[id]) + + def delete(self, req, id): + context = req.environ['nova.api_request_context'] + # TODO(gundlach): make sure it's an image they may delete? + return images.deregister(context, id) + + def create(self, **kwargs): + # TODO(gundlach): no idea how to hook this up. code below + # is from servers.py. + inst = self.build_server_instance(kwargs['server']) + rpc.cast( + FLAGS.compute_topic, { + "method": "run_instance", + "args": {"instance_id": inst.instance_id}}) + + def update(self, **kwargs): + # TODO (gundlach): no idea how to hook this up. code below + # is from servers.py. + instance_id = kwargs['id'] + instance = compute.InstanceDirectory().get(instance_id) + if not instance: + raise ServerNotFound("The requested server was not found") + instance.update(kwargs['server']) + instance.save() -- cgit From 567aa0ac862f0cb18786f20d949ab75bd800c3c7 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Thu, 19 Aug 2010 15:05:13 +0100 Subject: Remove whitespace to match style guide. --- nova/virt/xenapi.py | 15 --------------- 1 file changed, 15 deletions(-) (limited to 'nova') diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py index aed4c4fb5..f0bbbbe1f 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi.py @@ -149,7 +149,6 @@ class XenAPIConnection(object): yield self._call_xenapi('VM.start', vm_ref, False, False) logging.info('Spawning VM %s created %s.', instance.name, vm_ref) - @defer.inlineCallbacks def _create_vm(self, instance, kernel, ramdisk): """Create a VM record. Returns a Deferred that gives the new @@ -191,7 +190,6 @@ class XenAPIConnection(object): logging.debug('Created VM %s as %s.', instance.name, vm_ref) defer.returnValue(vm_ref) - @defer.inlineCallbacks def _create_vbd(self, vm_ref, vdi_ref, userdevice, bootable): """Create a VBD record. Returns a Deferred that gives the new @@ -216,7 +214,6 @@ class XenAPIConnection(object): vdi_ref) defer.returnValue(vbd_ref) - @defer.inlineCallbacks def _create_vif(self, vm_ref, network_ref, mac_address): """Create a VIF record. Returns a Deferred that gives the new @@ -238,7 +235,6 @@ class XenAPIConnection(object): vm_ref, network_ref) defer.returnValue(vif_ref) - @defer.inlineCallbacks def _find_network_with_bridge(self, bridge): expr = 'field "bridge" = "%s"' % bridge @@ -251,7 +247,6 @@ class XenAPIConnection(object): else: raise Exception('Found no network for bridge %s' % bridge) - @defer.inlineCallbacks def _fetch_image(self, image, user, project, use_sr): """use_sr: True to put the image as a VDI in an SR, False to place @@ -273,7 +268,6 @@ class XenAPIConnection(object): uuid = yield self._wait_for_task(task) defer.returnValue(uuid) - @defer.inlineCallbacks def reboot(self, instance): vm = yield self._lookup(instance.name) @@ -282,7 +276,6 @@ class XenAPIConnection(object): task = yield self._call_xenapi('Async.VM.clean_reboot', vm) yield self._wait_for_task(task) - @defer.inlineCallbacks def destroy(self, instance): vm = yield self._lookup(instance.name) @@ -291,7 +284,6 @@ class XenAPIConnection(object): task = yield self._call_xenapi('Async.VM.destroy', vm) yield self._wait_for_task(task) - def get_info(self, instance_id): vm = self._lookup_blocking(instance_id) if vm is None: @@ -303,12 +295,10 @@ class XenAPIConnection(object): 'num_cpu': rec['VCPUs_max'], 'cpu_time': 0} - @deferredToThread def _lookup(self, i): return self._lookup_blocking(i) - def _lookup_blocking(self, i): vms = self._conn.xenapi.VM.get_by_name_label(i) n = len(vms) @@ -319,7 +309,6 @@ class XenAPIConnection(object): else: return vms[0] - def _wait_for_task(self, task): """Return a Deferred that will give the result of the given task. The task is polled until it completes.""" @@ -327,7 +316,6 @@ class XenAPIConnection(object): reactor.callLater(0, self._poll_task, task, d) return d - @deferredToThread def _poll_task(self, task, deferred): """Poll the given XenAPI task, and fire the given Deferred if we @@ -352,7 +340,6 @@ class XenAPIConnection(object): logging.warn(exn) deferred.errback(exn) - @deferredToThread def _call_xenapi(self, method, *args): """Call the specified XenAPI method on a background thread. Returns @@ -362,7 +349,6 @@ class XenAPIConnection(object): f = f.__getattr__(m) return f(*args) - @deferredToThread def _async_call_plugin(self, plugin, fn, args): """Call Async.host.call_plugin on a background thread. Returns a @@ -371,7 +357,6 @@ class XenAPIConnection(object): self._conn.xenapi.Async.host.call_plugin, self._get_xenapi_host(), plugin, fn, args) - def _get_xenapi_host(self): return self._conn.xenapi.session.get_this_host(self._conn.handle) -- cgit From 4a23d5d9091823e9b4dc364383a14b566af80cd6 Mon Sep 17 00:00:00 2001 From: Ewan Mellor Date: Thu, 19 Aug 2010 15:12:46 +0100 Subject: Move deferredToThread into utils, as suggested by termie. --- nova/utils.py | 8 ++++++++ nova/virt/xenapi.py | 18 ++++++------------ 2 files changed, 14 insertions(+), 12 deletions(-) (limited to 'nova') diff --git a/nova/utils.py b/nova/utils.py index e826f9b71..b0d07af79 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -29,6 +29,8 @@ import subprocess import socket import sys +from twisted.internet.threads import deferToThread + from nova import exception from nova import flags @@ -142,3 +144,9 @@ def isotime(at=None): def parse_isotime(timestr): return datetime.datetime.strptime(timestr, TIME_FORMAT) + + +def deferredToThread(f): + def g(*args, **kwargs): + return deferToThread(f, *args, **kwargs) + return g diff --git a/nova/virt/xenapi.py b/nova/virt/xenapi.py index f0bbbbe1f..b44ac383a 100644 --- a/nova/virt/xenapi.py +++ b/nova/virt/xenapi.py @@ -19,7 +19,7 @@ A connection to XenServer or Xen Cloud Platform. The concurrency model for this class is as follows: -All XenAPI calls are on a thread (using t.i.t.deferToThread, or the decorator +All XenAPI calls are on a thread (using t.i.t.deferToThread, via the decorator deferredToThread). They are remote calls, and so may hang for the usual reasons. They should not be allowed to block the reactor thread. @@ -41,10 +41,10 @@ import xmlrpclib from twisted.internet import defer from twisted.internet import reactor from twisted.internet import task -from twisted.internet.threads import deferToThread from nova import flags from nova import process +from nova import utils from nova.auth.manager import AuthManager from nova.compute import power_state from nova.virt import images @@ -97,12 +97,6 @@ def get_connection(_): return XenAPIConnection(url, username, password) -def deferredToThread(f): - def g(*args, **kwargs): - return deferToThread(f, *args, **kwargs) - return g - - class XenAPIConnection(object): def __init__(self, url, user, pw): self._conn = XenAPI.Session(url) @@ -295,7 +289,7 @@ class XenAPIConnection(object): 'num_cpu': rec['VCPUs_max'], 'cpu_time': 0} - @deferredToThread + @utils.deferredToThread def _lookup(self, i): return self._lookup_blocking(i) @@ -316,7 +310,7 @@ class XenAPIConnection(object): reactor.callLater(0, self._poll_task, task, d) return d - @deferredToThread + @utils.deferredToThread def _poll_task(self, task, deferred): """Poll the given XenAPI task, and fire the given Deferred if we get a result.""" @@ -340,7 +334,7 @@ class XenAPIConnection(object): logging.warn(exn) deferred.errback(exn) - @deferredToThread + @utils.deferredToThread def _call_xenapi(self, method, *args): """Call the specified XenAPI method on a background thread. Returns a Deferred for the result.""" @@ -349,7 +343,7 @@ class XenAPIConnection(object): f = f.__getattr__(m) return f(*args) - @deferredToThread + @utils.deferredToThread def _async_call_plugin(self, plugin, fn, args): """Call Async.host.call_plugin on a background thread. Returns a Deferred with the task reference.""" -- cgit From b651008e7e4f60f2ccb07497c27d866814156209 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Thu, 19 Aug 2010 16:05:27 -0400 Subject: Complete the Image API against a LocalImageService until Glance's API exists (at which point we'll make a GlanceImageService and make the choice of ImageService plugin configurable.) --- nova/api/rackspace/images.py | 83 +++++++++++++++++++++++++++++-------------- nova/api/rackspace/notes.txt | 23 ++++++++++++ nova/api/services/__init__.py | 0 nova/api/services/image.py | 72 +++++++++++++++++++++++++++++++++++++ 4 files changed, 151 insertions(+), 27 deletions(-) create mode 100644 nova/api/rackspace/notes.txt create mode 100644 nova/api/services/__init__.py create mode 100644 nova/api/services/image.py (limited to 'nova') diff --git a/nova/api/rackspace/images.py b/nova/api/rackspace/images.py index 57c03894a..e29f737a5 100644 --- a/nova/api/rackspace/images.py +++ b/nova/api/rackspace/images.py @@ -15,12 +15,13 @@ # License for the specific language governing permissions and limitations # under the License. -from nova.endpoint.rackspace.controllers.base import BaseController -from nova.endpoint import images +from nova import datastore +from nova.api.rackspace import base +from nova.api.services.image import ImageService from webob import exc #TODO(gundlach): Serialize return values -class Controller(BaseController): +class Controller(base.Controller): _serialization_metadata = { 'application/xml': { @@ -31,34 +32,62 @@ class Controller(BaseController): } } + def __init__(self): + self._svc = ImageService.load() + self._id_xlator = RackspaceApiImageIdTranslator() + + def _to_rs_id(self, image_id): + """ + Convert an image id from the format of our ImageService strategy + to the Rackspace API format (an int). + """ + strategy = self._svc.__class__.__name__ + return self._id_xlator.to_rs_id(strategy, image_id) + def index(self, req): - context = req.environ['nova.api_request_context'] - return images.list(context) + """Return all public images.""" + data = self._svc.list() + for img in data: + img['id'] = self._to_rs_id(img['id']) + return dict(images=result) def show(self, req, id): - context = req.environ['nova.api_request_context'] - return images.list(context, filter_list=[id]) + """Return data about the given image id.""" + img = self._svc.show(id) + img['id'] = self._to_rs_id(img['id']) + return dict(image=img) def delete(self, req, id): - context = req.environ['nova.api_request_context'] - # TODO(gundlach): make sure it's an image they may delete? - return images.deregister(context, id) + # Only public images are supported for now. + raise exc.HTTPNotFound() + + def create(self, req): + # Only public images are supported for now, so a request to + # make a backup of a server cannot be supproted. + raise exc.HTTPNotFound() + + def update(self, req, id): + # Users may not modify public images, and that's all that + # we support for now. + raise exc.HTTPNotFound() + + +class RackspaceApiImageIdTranslator(object): + """ + Converts Rackspace API image ids to and from the id format for a given + strategy. + """ - def create(self, **kwargs): - # TODO(gundlach): no idea how to hook this up. code below - # is from servers.py. - inst = self.build_server_instance(kwargs['server']) - rpc.cast( - FLAGS.compute_topic, { - "method": "run_instance", - "args": {"instance_id": inst.instance_id}}) + def __init__(self): + self._store = datastore.Redis.instance() - def update(self, **kwargs): - # TODO (gundlach): no idea how to hook this up. code below - # is from servers.py. - instance_id = kwargs['id'] - instance = compute.InstanceDirectory().get(instance_id) - if not instance: - raise ServerNotFound("The requested server was not found") - instance.update(kwargs['server']) - instance.save() + def to_rs_id(self, strategy_name, opaque_id): + """Convert an id from a strategy-specific one to a Rackspace one.""" + key = "rsapi.idstrategies.image.%s" % strategy_name + result = self._store.hget(key, str(opaque_id)) + if result: # we have a mapping from opaque to RS for this strategy + return int(result) + else: + nextid = self._store.incr("%s.lastid" % key) + self._store.hsetnx(key, str(opaque_id), nextid) + return nextid diff --git a/nova/api/rackspace/notes.txt b/nova/api/rackspace/notes.txt new file mode 100644 index 000000000..e133bf5ea --- /dev/null +++ b/nova/api/rackspace/notes.txt @@ -0,0 +1,23 @@ +We will need: + +ImageService +a service that can do crud on image information. not user-specific. opaque +image ids. + +GlanceImageService(ImageService): +image ids are URIs. + +LocalImageService(ImageService): +image ids are random strings. + +RackspaceAPITranslationStore: +translates RS server/images/flavor/etc ids into formats required +by a given ImageService strategy. + +api.rackspace.images.Controller: +uses an ImageService strategy behind the scenes to do its fetching; it just +converts int image id into a strategy-specific image id. + +who maintains the mapping from user to [images he owns]? nobody, because +we have no way of enforcing access to his images, without kryptex which +won't be in Austin. diff --git a/nova/api/services/__init__.py b/nova/api/services/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/nova/api/services/image.py b/nova/api/services/image.py new file mode 100644 index 000000000..c5ea15ba1 --- /dev/null +++ b/nova/api/services/image.py @@ -0,0 +1,72 @@ +import cPickle as pickle +import os.path +import string + +class ImageService(object): + """Provides storage and retrieval of disk image objects.""" + + @staticmethod + def load(): + """Factory method to return image service.""" + #TODO(gundlach): read from config. + class_ = LocalImageService + return class_() + + def index(self): + """ + Return a list of image data dicts. Each dict will contain an + id key whose value is an opaque image id. + """ + + def show(self, id): + """ + Returns a dict containing image data for the given opaque image id. + """ + + +class GlanceImageService(ImageService): + """Provides storage and retrieval of disk image objects within Glance.""" + # TODO(gundlach): once Glance has an API, build this. + pass + + +class LocalImageService(ImageService): + """Image service storing images to local disk.""" + + def __init__(self): + self._path = "/tmp/nova/images" + try: + os.makedirs(self._path) + except OSError: # exists + pass + + def _path_to(self, image_id=''): + return os.path.join(self._path, image_id) + + def _ids(self): + """The list of all image ids.""" + return os.path.listdir(self._path) + + def index(self): + return [ self.show(id) for id in self._ids() ] + + def show(self, id): + return pickle.load(open(self._path_to(id))) + + def create(self, data): + """ + Store the image data and return the new image id. + """ + id = ''.join(random.choice(string.letters) for _ in range(20)) + self.update(id, data) + return id + + def update(self, image_id, data): + """Replace the contents of the given image with the new data.""" + pickle.dump(data, open(self._path_to(image_id), 'w')) + + def delete(self, image_id): + """ + Delete the given image. Raises OSError if the image does not exist. + """ + os.unlink(self._path_to(image_id)) -- cgit From a39a155342ad5aa9d8c7b115fb6fe7498ef00f23 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Fri, 20 Aug 2010 10:08:05 -0700 Subject: small fixes to network --- nova/network/service.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'nova') diff --git a/nova/network/service.py b/nova/network/service.py index d3aa1c46f..3dba0a9ef 100644 --- a/nova/network/service.py +++ b/nova/network/service.py @@ -26,7 +26,7 @@ from nova import flags from nova import service from nova import utils from nova.auth import manager -from nova.network import exception +from nova.network import exception as network_exception from nova.network import model from nova.network import vpn @@ -64,8 +64,7 @@ def type_to_class(network_type): def setup_compute_network(network_type, user_id, project_id, security_group): """Sets up the network on a compute host""" srv = type_to_class(network_type) - srv.setup_compute_network(network_type, - user_id, + srv.setup_compute_network(user_id, project_id, security_group) @@ -170,7 +169,7 @@ class FlatNetworkService(BaseNetworkService): redis.sadd('ips', fixed_ip) fixed_ip = redis.spop('ips') if not fixed_ip: - raise exception.NoMoreAddresses() + raise network_exception.NoMoreAddresses() # TODO(vish): some sort of dns handling for hostname should # probably be done here. return {'inject_network': True, -- cgit From d38f21e0fb382bd8f01cfbc79cb34ea8710cd639 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 23 Aug 2010 10:27:59 -0400 Subject: License --- nova/api/services/image.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'nova') diff --git a/nova/api/services/image.py b/nova/api/services/image.py index c5ea15ba1..bda50fc66 100644 --- a/nova/api/services/image.py +++ b/nova/api/services/image.py @@ -1,3 +1,20 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + import cPickle as pickle import os.path import string -- cgit From e3727d6d88a0631d3b896c4fcdcfec05510dad36 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 23 Aug 2010 12:07:20 -0400 Subject: Support opaque id to rs int id as well --- nova/api/rackspace/images.py | 42 ++++++++++++++++++++++++++++++++---------- nova/api/services/image.py | 8 ++++---- 2 files changed, 36 insertions(+), 14 deletions(-) (limited to 'nova') diff --git a/nova/api/rackspace/images.py b/nova/api/rackspace/images.py index e29f737a5..c9cc8e85d 100644 --- a/nova/api/rackspace/images.py +++ b/nova/api/rackspace/images.py @@ -44,18 +44,24 @@ class Controller(base.Controller): strategy = self._svc.__class__.__name__ return self._id_xlator.to_rs_id(strategy, image_id) + def _from_rs_id(self, rs_image_id): + """ + Convert an image id from the Rackspace API format (an int) to the + format of our ImageService strategy. + """ + strategy = self._svc.__class__.__name__ + return self._id_xlator.from_rs_id(strategy, rs_image_id) + def index(self, req): """Return all public images.""" - data = self._svc.list() - for img in data: - img['id'] = self._to_rs_id(img['id']) - return dict(images=result) + data = dict((self._to_rs_id(id), val) + for id, val in self._svc.index().iteritems()) + return dict(images=data) def show(self, req, id): """Return data about the given image id.""" - img = self._svc.show(id) - img['id'] = self._to_rs_id(img['id']) - return dict(image=img) + opaque_id = self._from_rs_id(id) + return dict(image=self._svc.show(opaque_id)) def delete(self, req, id): # Only public images are supported for now. @@ -80,14 +86,30 @@ class RackspaceApiImageIdTranslator(object): def __init__(self): self._store = datastore.Redis.instance() + self._key_template = "rsapi.idstrategies.image.%s.%s" def to_rs_id(self, strategy_name, opaque_id): """Convert an id from a strategy-specific one to a Rackspace one.""" - key = "rsapi.idstrategies.image.%s" % strategy_name + key = self._key_template % (strategy_name, "fwd") result = self._store.hget(key, str(opaque_id)) if result: # we have a mapping from opaque to RS for this strategy return int(result) else: + # Store the mapping. nextid = self._store.incr("%s.lastid" % key) - self._store.hsetnx(key, str(opaque_id), nextid) - return nextid + if self._store.hsetnx(key, str(opaque_id), nextid): + # If someone else didn't beat us to it, store the reverse + # mapping as well. + key = self._key_template % (strategy_name, "rev") + self._store.hset(key, nextid, str(opaque_id)) + return nextid + else: + # Someone beat us to it; use their number instead, and + # discard nextid (which is OK -- we don't require that + # every int id be used.) + return int(self._store.hget(key, str(opaque_id))) + + def from_rs_id(self, strategy_name, rs_id): + """Convert a Rackspace id to a strategy-specific one.""" + key = self._key_template % (strategy_name, "rev") + return self._store.hget(key, rs_id) diff --git a/nova/api/services/image.py b/nova/api/services/image.py index bda50fc66..11e19804a 100644 --- a/nova/api/services/image.py +++ b/nova/api/services/image.py @@ -17,6 +17,7 @@ import cPickle as pickle import os.path +import random import string class ImageService(object): @@ -31,8 +32,7 @@ class ImageService(object): def index(self): """ - Return a list of image data dicts. Each dict will contain an - id key whose value is an opaque image id. + Return a dict from opaque image id to image data. """ def show(self, id): @@ -62,10 +62,10 @@ class LocalImageService(ImageService): def _ids(self): """The list of all image ids.""" - return os.path.listdir(self._path) + return os.listdir(self._path) def index(self): - return [ self.show(id) for id in self._ids() ] + return dict((id, self.show(id)) for id in self._ids()) def show(self, id): return pickle.load(open(self._path_to(id))) -- cgit From 030d01fd10f7f65cdafbea49e04f3b6b147a7348 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 23 Aug 2010 12:46:29 -0400 Subject: Serialize properly --- nova/api/rackspace/base.py | 3 ++- nova/api/rackspace/images.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/api/rackspace/base.py b/nova/api/rackspace/base.py index c85fd7b8e..b995d9acc 100644 --- a/nova/api/rackspace/base.py +++ b/nova/api/rackspace/base.py @@ -36,4 +36,5 @@ class Controller(wsgi.Controller): MIME types to information needed to serialize to that type. """ _metadata = getattr(type(self), "_serialization_metadata", {}) - return Serializer(request.environ, _metadata).to_content_type(data) + serializer = wsgi.Serializer(request.environ, _metadata) + return serializer.to_content_type(data) diff --git a/nova/api/rackspace/images.py b/nova/api/rackspace/images.py index c9cc8e85d..62e0b24c5 100644 --- a/nova/api/rackspace/images.py +++ b/nova/api/rackspace/images.py @@ -56,12 +56,12 @@ class Controller(base.Controller): """Return all public images.""" data = dict((self._to_rs_id(id), val) for id, val in self._svc.index().iteritems()) - return dict(images=data) + return self.serialize(dict(images=data), req) def show(self, req, id): """Return data about the given image id.""" opaque_id = self._from_rs_id(id) - return dict(image=self._svc.show(opaque_id)) + return self.serialize(dict(image=self._svc.show(opaque_id)), req) def delete(self, req, id): # Only public images are supported for now. -- cgit From a50a200bc2547439a3da17e695224d3d434e14dd Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 23 Aug 2010 12:55:57 -0400 Subject: Move serialize() to wsgi.Controller so __call__ can serialize() action return values if they are dicts. --- nova/api/rackspace/base.py | 10 ---------- nova/wsgi.py | 16 ++++++++++++++-- 2 files changed, 14 insertions(+), 12 deletions(-) (limited to 'nova') diff --git a/nova/api/rackspace/base.py b/nova/api/rackspace/base.py index b995d9acc..51841925e 100644 --- a/nova/api/rackspace/base.py +++ b/nova/api/rackspace/base.py @@ -28,13 +28,3 @@ class Controller(wsgi.Controller): return {cls.entity_name: cls.render(instance)} else: return { "TODO": "TODO" } - - def serialize(self, data, request): - """ - Serialize the given dict to the response type requested in request. - Uses self._serialization_metadata if it exists, which is a dict mapping - MIME types to information needed to serialize to that type. - """ - _metadata = getattr(type(self), "_serialization_metadata", {}) - serializer = wsgi.Serializer(request.environ, _metadata) - return serializer.to_content_type(data) diff --git a/nova/wsgi.py b/nova/wsgi.py index baf6cccd9..d52bf855d 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -196,7 +196,8 @@ class Controller(object): WSGI app that reads routing information supplied by RoutesMiddleware and calls the requested action method upon itself. All action methods must, in addition to their normal parameters, accept a 'req' argument - which is the incoming webob.Request. + which is the incoming webob.Request. They raise a webob.exc exception, + or return a dict which will be serialized by requested content type. """ @webob.dec.wsgify @@ -210,7 +211,18 @@ class Controller(object): del arg_dict['controller'] del arg_dict['action'] arg_dict['req'] = req - return method(**arg_dict) + result = method(**arg_dict) + return self._serialize(result, req) if type(result) is dict else result + + def _serialize(self, data, request): + """ + Serialize the given dict to the response type requested in request. + Uses self._serialization_metadata if it exists, which is a dict mapping + MIME types to information needed to serialize to that type. + """ + _metadata = getattr(type(self), "_serialization_metadata", {}) + serializer = wsgi.Serializer(request.environ, _metadata) + return serializer.to_content_type(data) class Serializer(object): -- cgit From f5c03fdd78a3bb8233e465c7624ed1fdb8f400fe Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 23 Aug 2010 13:06:40 -0400 Subject: Don't serialize in Controller subclass now that wsgi.Controller handles it for us --- nova/api/rackspace/images.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'nova') diff --git a/nova/api/rackspace/images.py b/nova/api/rackspace/images.py index 62e0b24c5..070100143 100644 --- a/nova/api/rackspace/images.py +++ b/nova/api/rackspace/images.py @@ -20,7 +20,6 @@ from nova.api.rackspace import base from nova.api.services.image import ImageService from webob import exc -#TODO(gundlach): Serialize return values class Controller(base.Controller): _serialization_metadata = { @@ -56,12 +55,12 @@ class Controller(base.Controller): """Return all public images.""" data = dict((self._to_rs_id(id), val) for id, val in self._svc.index().iteritems()) - return self.serialize(dict(images=data), req) + return dict(images=data) def show(self, req, id): """Return data about the given image id.""" opaque_id = self._from_rs_id(id) - return self.serialize(dict(image=self._svc.show(opaque_id)), req) + return dict(image=self._svc.show(opaque_id)) def delete(self, req, id): # Only public images are supported for now. -- cgit From c49c725e43cfbc9d90b5e9ebbf93a32e71c7e6a9 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 23 Aug 2010 13:07:43 -0400 Subject: Typo --- nova/wsgi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/wsgi.py b/nova/wsgi.py index d52bf855d..096d5843f 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -221,7 +221,7 @@ class Controller(object): MIME types to information needed to serialize to that type. """ _metadata = getattr(type(self), "_serialization_metadata", {}) - serializer = wsgi.Serializer(request.environ, _metadata) + serializer = Serializer(request.environ, _metadata) return serializer.to_content_type(data) -- cgit From 35a08780c41ece1b47b2ded98c061b103a400fea Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 23 Aug 2010 13:26:10 -0400 Subject: Get the output formatting correct. --- nova/api/rackspace/images.py | 9 ++++++--- nova/api/services/image.py | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) (limited to 'nova') diff --git a/nova/api/rackspace/images.py b/nova/api/rackspace/images.py index 070100143..7d32fa099 100644 --- a/nova/api/rackspace/images.py +++ b/nova/api/rackspace/images.py @@ -53,14 +53,17 @@ class Controller(base.Controller): def index(self, req): """Return all public images.""" - data = dict((self._to_rs_id(id), val) - for id, val in self._svc.index().iteritems()) + data = self._svc.index() + for img in data: + img['id'] = self._to_rs_id(img['id']) return dict(images=data) def show(self, req, id): """Return data about the given image id.""" opaque_id = self._from_rs_id(id) - return dict(image=self._svc.show(opaque_id)) + img = self._svc.show(opaque_id) + img['id'] = id + return dict(image=img) def delete(self, req, id): # Only public images are supported for now. diff --git a/nova/api/services/image.py b/nova/api/services/image.py index 11e19804a..1a7a258b7 100644 --- a/nova/api/services/image.py +++ b/nova/api/services/image.py @@ -65,7 +65,7 @@ class LocalImageService(ImageService): return os.listdir(self._path) def index(self): - return dict((id, self.show(id)) for id in self._ids()) + return [ self.show(id) for id in self._ids() ] def show(self, id): return pickle.load(open(self._path_to(id))) @@ -75,6 +75,7 @@ class LocalImageService(ImageService): Store the image data and return the new image id. """ id = ''.join(random.choice(string.letters) for _ in range(20)) + data['id'] = id self.update(id, data) return id -- cgit From 41e2e91ccfb1409f1ea47d92a9d15f47ab37e65d Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Mon, 23 Aug 2010 16:43:25 -0400 Subject: Merge fail --- nova/api/rackspace/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/api/rackspace/base.py b/nova/api/rackspace/base.py index 51841925e..dd2c6543c 100644 --- a/nova/api/rackspace/base.py +++ b/nova/api/rackspace/base.py @@ -27,4 +27,4 @@ class Controller(wsgi.Controller): if isinstance(instance, list): return {cls.entity_name: cls.render(instance)} else: - return { "TODO": "TODO" } + return {"TODO": "TODO"} -- cgit From d94eec3d2995c97c38006e4d6177740375860f8f Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 24 Aug 2010 11:19:51 -0400 Subject: Style fixes --- nova/api/rackspace/images.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) (limited to 'nova') diff --git a/nova/api/rackspace/images.py b/nova/api/rackspace/images.py index 7d32fa099..36a26688c 100644 --- a/nova/api/rackspace/images.py +++ b/nova/api/rackspace/images.py @@ -17,7 +17,7 @@ from nova import datastore from nova.api.rackspace import base -from nova.api.services.image import ImageService +from nova.api.services import image from webob import exc class Controller(base.Controller): @@ -32,28 +32,28 @@ class Controller(base.Controller): } def __init__(self): - self._svc = ImageService.load() - self._id_xlator = RackspaceApiImageIdTranslator() + self._service = image.ImageService.load() + self._id_translator = RackspaceAPIImageIdTranslator() def _to_rs_id(self, image_id): """ Convert an image id from the format of our ImageService strategy to the Rackspace API format (an int). """ - strategy = self._svc.__class__.__name__ - return self._id_xlator.to_rs_id(strategy, image_id) + strategy = self._service.__class__.__name__ + return self._id_translator.to_rs_id(strategy, image_id) def _from_rs_id(self, rs_image_id): """ Convert an image id from the Rackspace API format (an int) to the format of our ImageService strategy. """ - strategy = self._svc.__class__.__name__ - return self._id_xlator.from_rs_id(strategy, rs_image_id) + strategy = self._service.__class__.__name__ + return self._id_translator.from_rs_id(strategy, rs_image_id) def index(self, req): """Return all public images.""" - data = self._svc.index() + data = self._service.index() for img in data: img['id'] = self._to_rs_id(img['id']) return dict(images=data) @@ -61,7 +61,7 @@ class Controller(base.Controller): def show(self, req, id): """Return data about the given image id.""" opaque_id = self._from_rs_id(id) - img = self._svc.show(opaque_id) + img = self._service.show(opaque_id) img['id'] = id return dict(image=img) @@ -80,7 +80,7 @@ class Controller(base.Controller): raise exc.HTTPNotFound() -class RackspaceApiImageIdTranslator(object): +class RackspaceAPIImageIdTranslator(object): """ Converts Rackspace API image ids to and from the id format for a given strategy. -- cgit From 4d1b2539d2d2f39ca53e9383e317af76dbc71905 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 24 Aug 2010 13:37:18 -0400 Subject: OK, break out ternary operator (good to know that it slowed you down to read it) --- nova/wsgi.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/wsgi.py b/nova/wsgi.py index 096d5843f..bec0a7b1c 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -212,7 +212,10 @@ class Controller(object): del arg_dict['action'] arg_dict['req'] = req result = method(**arg_dict) - return self._serialize(result, req) if type(result) is dict else result + if type(result) is dict: + return self._serialize(result, req) + else: + return result def _serialize(self, data, request): """ -- cgit From 09bc71460b976f28c7bc6a507006d6c7c12c5824 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 24 Aug 2010 16:16:41 -0400 Subject: Move imageservice to its own directory --- nova/api/rackspace/images.py | 4 +- nova/api/services/image.py | 90 -------------------------------------------- nova/image/service.py | 90 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 92 insertions(+), 92 deletions(-) delete mode 100644 nova/api/services/image.py create mode 100644 nova/image/service.py (limited to 'nova') diff --git a/nova/api/rackspace/images.py b/nova/api/rackspace/images.py index 36a26688c..b7668a1e1 100644 --- a/nova/api/rackspace/images.py +++ b/nova/api/rackspace/images.py @@ -17,7 +17,7 @@ from nova import datastore from nova.api.rackspace import base -from nova.api.services import image +from nova import image from webob import exc class Controller(base.Controller): @@ -32,7 +32,7 @@ class Controller(base.Controller): } def __init__(self): - self._service = image.ImageService.load() + self._service = image.service.ImageService.load() self._id_translator = RackspaceAPIImageIdTranslator() def _to_rs_id(self, image_id): diff --git a/nova/api/services/image.py b/nova/api/services/image.py deleted file mode 100644 index 1a7a258b7..000000000 --- a/nova/api/services/image.py +++ /dev/null @@ -1,90 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2010 OpenStack LLC. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import cPickle as pickle -import os.path -import random -import string - -class ImageService(object): - """Provides storage and retrieval of disk image objects.""" - - @staticmethod - def load(): - """Factory method to return image service.""" - #TODO(gundlach): read from config. - class_ = LocalImageService - return class_() - - def index(self): - """ - Return a dict from opaque image id to image data. - """ - - def show(self, id): - """ - Returns a dict containing image data for the given opaque image id. - """ - - -class GlanceImageService(ImageService): - """Provides storage and retrieval of disk image objects within Glance.""" - # TODO(gundlach): once Glance has an API, build this. - pass - - -class LocalImageService(ImageService): - """Image service storing images to local disk.""" - - def __init__(self): - self._path = "/tmp/nova/images" - try: - os.makedirs(self._path) - except OSError: # exists - pass - - def _path_to(self, image_id=''): - return os.path.join(self._path, image_id) - - def _ids(self): - """The list of all image ids.""" - return os.listdir(self._path) - - def index(self): - return [ self.show(id) for id in self._ids() ] - - def show(self, id): - return pickle.load(open(self._path_to(id))) - - def create(self, data): - """ - Store the image data and return the new image id. - """ - id = ''.join(random.choice(string.letters) for _ in range(20)) - data['id'] = id - self.update(id, data) - return id - - def update(self, image_id, data): - """Replace the contents of the given image with the new data.""" - pickle.dump(data, open(self._path_to(image_id), 'w')) - - def delete(self, image_id): - """ - Delete the given image. Raises OSError if the image does not exist. - """ - os.unlink(self._path_to(image_id)) diff --git a/nova/image/service.py b/nova/image/service.py new file mode 100644 index 000000000..1a7a258b7 --- /dev/null +++ b/nova/image/service.py @@ -0,0 +1,90 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2010 OpenStack LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import cPickle as pickle +import os.path +import random +import string + +class ImageService(object): + """Provides storage and retrieval of disk image objects.""" + + @staticmethod + def load(): + """Factory method to return image service.""" + #TODO(gundlach): read from config. + class_ = LocalImageService + return class_() + + def index(self): + """ + Return a dict from opaque image id to image data. + """ + + def show(self, id): + """ + Returns a dict containing image data for the given opaque image id. + """ + + +class GlanceImageService(ImageService): + """Provides storage and retrieval of disk image objects within Glance.""" + # TODO(gundlach): once Glance has an API, build this. + pass + + +class LocalImageService(ImageService): + """Image service storing images to local disk.""" + + def __init__(self): + self._path = "/tmp/nova/images" + try: + os.makedirs(self._path) + except OSError: # exists + pass + + def _path_to(self, image_id=''): + return os.path.join(self._path, image_id) + + def _ids(self): + """The list of all image ids.""" + return os.listdir(self._path) + + def index(self): + return [ self.show(id) for id in self._ids() ] + + def show(self, id): + return pickle.load(open(self._path_to(id))) + + def create(self, data): + """ + Store the image data and return the new image id. + """ + id = ''.join(random.choice(string.letters) for _ in range(20)) + data['id'] = id + self.update(id, data) + return id + + def update(self, image_id, data): + """Replace the contents of the given image with the new data.""" + pickle.dump(data, open(self._path_to(image_id), 'w')) + + def delete(self, image_id): + """ + Delete the given image. Raises OSError if the image does not exist. + """ + os.unlink(self._path_to(image_id)) -- cgit From 5f832cd5ea9fb858f5e8b09992cbd47d8d16f665 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 24 Aug 2010 16:17:06 -0400 Subject: Delete unused directory --- nova/api/services/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 nova/api/services/__init__.py (limited to 'nova') diff --git a/nova/api/services/__init__.py b/nova/api/services/__init__.py deleted file mode 100644 index e69de29bb..000000000 -- cgit From 96ae5e2776218adfee2cb22a4c0d7358a498a451 Mon Sep 17 00:00:00 2001 From: Michael Gundlach Date: Tue, 24 Aug 2010 16:24:24 -0400 Subject: pep8 --- nova/api/rackspace/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova') diff --git a/nova/api/rackspace/images.py b/nova/api/rackspace/images.py index b7668a1e1..370980fe9 100644 --- a/nova/api/rackspace/images.py +++ b/nova/api/rackspace/images.py @@ -16,8 +16,8 @@ # under the License. from nova import datastore -from nova.api.rackspace import base from nova import image +from nova.api.rackspace import base from webob import exc class Controller(base.Controller): -- cgit