From 38c21546ecc079300c575e5950bcb990eecee3a3 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 27 Feb 2011 20:28:04 -0500 Subject: execute: shell=True removed. --- nova/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 0cf91e0cc..40a8d8d8c 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -125,7 +125,7 @@ def fetchfile(url, target): # c.perform() # c.close() # fp.close() - execute("curl --fail %s -o %s" % (url, target)) + execute("curl","--fail",url,"-o",target) def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): @@ -133,7 +133,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + obj = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) result = None if process_input != None: @@ -254,7 +254,7 @@ def last_octet(address): def get_my_linklocal(interface): try: - if_str = execute("ip -f inet6 -o addr show %s" % interface) + if_str = execute("ip","-f","inet6","-o","addr","show", interface) condition = "\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link" links = [re.search(condition, x) for x in if_str[0].split('\n')] address = [w.group(1) for w in links if w is not None] -- cgit From 4f90783224025618661bf8814e016843ec237875 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 27 Feb 2011 20:52:32 -0500 Subject: execvp --- nova/volume/driver.py | 69 ++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 34 deletions(-) diff --git a/nova/volume/driver.py b/nova/volume/driver.py index e3744c790..e73202b73 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -97,22 +97,21 @@ class VolumeDriver(object): sizestr = '100M' else: sizestr = '%sG' % volume['size'] - self._try_execute("sudo lvcreate -L %s -n %s %s" % - (sizestr, + self._try_execute('sudo','lvcreate','-L',sizestr,'-n', volume['name'], - FLAGS.volume_group)) + FLAGS.volume_group) def delete_volume(self, volume): """Deletes a logical volume.""" try: - self._try_execute("sudo lvdisplay %s/%s" % + self._try_execute('sudo','lvdisplay','%s/%s" % (FLAGS.volume_group, volume['name'])) except Exception as e: # If the volume isn't present, then don't attempt to delete return True - self._try_execute("sudo lvremove -f %s/%s" % + self._try_execute('sudo','lvremove','-f',"%s/%s" % (FLAGS.volume_group, volume['name'])) @@ -168,12 +167,13 @@ class AOEDriver(VolumeDriver): blade_id) = self.db.volume_allocate_shelf_and_blade(context, volume['id']) self._try_execute( - "sudo vblade-persist setup %s %s %s /dev/%s/%s" % - (shelf_id, + 'sudo','vblade-persist','setup', + shelf_id, blade_id, FLAGS.aoe_eth_dev, - FLAGS.volume_group, - volume['name'])) + "/dev/%s/%s" % + (FLAGS.volume_group, + volume['name'])) # NOTE(vish): The standard _try_execute does not work here # because these methods throw errors if other # volumes on this host are in the process of @@ -182,9 +182,9 @@ class AOEDriver(VolumeDriver): # just wait a bit for the current volume to # be ready and ignore any errors. time.sleep(2) - self._execute("sudo vblade-persist auto all", + self._execute('sudo','vblade-persist','auto','all', check_exit_code=False) - self._execute("sudo vblade-persist start all", + self._execute('sudo','vblade-persist','start','all', check_exit_code=False) def remove_export(self, context, volume): @@ -192,15 +192,15 @@ class AOEDriver(VolumeDriver): (shelf_id, blade_id) = self.db.volume_get_shelf_and_blade(context, volume['id']) - self._try_execute("sudo vblade-persist stop %s %s" % - (shelf_id, blade_id)) - self._try_execute("sudo vblade-persist destroy %s %s" % - (shelf_id, blade_id)) + self._try_execute('sudo','vblade-persist','stop', + shelf_id, blade_id) + self._try_execute('sudo','vblade-persist','destroy', + shelf_id, blade_id) def discover_volume(self, _volume): """Discover volume on a remote host.""" - self._execute("sudo aoe-discover") - self._execute("sudo aoe-stat", check_exit_code=False) + self._execute('sudo','aoe-discover') + self._execute('sudo','aoe-stat', check_exit_code=False) def undiscover_volume(self, _volume): """Undiscover volume on a remote host.""" @@ -252,13 +252,16 @@ class ISCSIDriver(VolumeDriver): iscsi_name = "%s%s" % (FLAGS.iscsi_target_prefix, volume['name']) volume_path = "/dev/%s/%s" % (FLAGS.volume_group, volume['name']) - self._sync_exec("sudo ietadm --op new " - "--tid=%s --params Name=%s" % - (iscsi_target, iscsi_name), + self._sync_exec('sudo','ietadm','--op','new', + "--tid=%s" % iscsi_target, + '--params', + "Name=%s" % iscsi-name, check_exit_code=False) - self._sync_exec("sudo ietadm --op new --tid=%s " - "--lun=0 --params Path=%s,Type=fileio" % - (iscsi_target, volume_path), + self._sync_exec('sudo','ietadm','--op','new', + "--tid=%s" % iscsi_target, + '--lun=0', + '--params', + "Path=%s,Type=fileio" % volume_path, check_exit_code=False) def _ensure_iscsi_targets(self, context, host): @@ -490,16 +493,13 @@ class RBDDriver(VolumeDriver): size = 100 else: size = int(volume['size']) * 1024 - self._try_execute("rbd --pool %s --size %d create %s" % - (FLAGS.rbd_pool, - size, - volume['name'])) + self._try_execute('rbd','--pool',FLAGS.rbd_pool, + '--size', size,'create', volume['name']) def delete_volume(self, volume): """Deletes a logical volume.""" - self._try_execute("rbd --pool %s rm %s" % - (FLAGS.rbd_pool, - volume['name'])) + self._try_execute('rbd','--pool',FLAGS.rbd_pool, + 'rm', voluname['name']) def local_path(self, volume): """Returns the path of the rbd volume.""" @@ -534,7 +534,7 @@ class SheepdogDriver(VolumeDriver): def check_for_setup_error(self): """Returns an error if prerequisites aren't met""" try: - (out, err) = self._execute("collie cluster info") + (out, err) = self._execute('collie','cluster','info') if not out.startswith('running'): raise exception.Error(_("Sheepdog is not working: %s") % out) except exception.ProcessExecutionError: @@ -546,12 +546,13 @@ class SheepdogDriver(VolumeDriver): sizestr = '100M' else: sizestr = '%sG' % volume['size'] - self._try_execute("qemu-img create sheepdog:%s %s" % - (volume['name'], sizestr)) + self._try_execute('qemu-img','create', + "sheepdog:%s" %s" % volume['name'], + sizestr) def delete_volume(self, volume): """Deletes a logical volume""" - self._try_execute("collie vdi delete %s" % volume['name']) + self._try_execute('collie','vdi','delete',volume['name']) def local_path(self, volume): return "sheepdog:%s" % volume['name'] -- cgit From 953efce36b74c18a32ef9c42e6b1a57190e3ff6e Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 27 Feb 2011 20:53:53 -0500 Subject: execvp --- nova/crypto.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/nova/crypto.py b/nova/crypto.py index a34b940f5..b240a3958 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -105,8 +105,8 @@ def generate_key_pair(bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.join(tmpdir, 'temp') - utils.execute('ssh-keygen -q -b %d -N "" -f %s' % (bits, keyfile)) - (out, err) = utils.execute('ssh-keygen -q -l -f %s.pub' % (keyfile)) + utils.execute('ssh-keygen','-q','-b',"%d" % bits,'-N','""','-f',keyfile) + (out, err) = utils.execute('ssh-keygen','-q','-l','-f',"%s.pub" % (keyfile)) fingerprint = out.split(' ')[1] private_key = open(keyfile).read() public_key = open(keyfile + '.pub').read() @@ -118,7 +118,7 @@ def generate_key_pair(bits=1024): # bio = M2Crypto.BIO.MemoryBuffer() # key.save_pub_key_bio(bio) # public_key = bio.read() - # public_key, err = execute('ssh-keygen -y -f /dev/stdin', private_key) + # public_key, err = execute('ssh-keygen','-y','-f','/dev/stdin', private_key) return (private_key, public_key, fingerprint) @@ -143,8 +143,8 @@ def revoke_cert(project_id, file_name): start = os.getcwd() os.chdir(ca_folder(project_id)) # NOTE(vish): potential race condition here - utils.execute("openssl ca -config ./openssl.cnf -revoke '%s'" % file_name) - utils.execute("openssl ca -gencrl -config ./openssl.cnf -out '%s'" % + utils.execute('openssl','ca','-config','./openssl.cnf','-revoke',"'%s'" % file_name) + utils.execute('openssl','ca','-gencrl','-config','./openssl.cnf','-out',"'%s'" % FLAGS.crl_file) os.chdir(start) @@ -193,9 +193,8 @@ def generate_x509_cert(user_id, project_id, bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key')) csrfile = os.path.join(tmpdir, 'temp.csr') - utils.execute("openssl genrsa -out %s %s" % (keyfile, bits)) - utils.execute("openssl req -new -key %s -out %s -batch -subj %s" % - (keyfile, csrfile, subject)) + utils.execute('openssl','genrsa','-out',keyfile,bits) + utils.execute('openssl','req','-new','-key',keyfile,'-out',csrfile,'-batch','-subj',subject) private_key = open(keyfile).read() csr = open(csrfile).read() shutil.rmtree(tmpdir) @@ -212,8 +211,7 @@ def _ensure_project_folder(project_id): if not os.path.exists(ca_path(project_id)): start = os.getcwd() os.chdir(ca_folder()) - utils.execute("sh geninter.sh %s %s" % - (project_id, _project_cert_subject(project_id))) + utils.execute('sh','geninter.sh',project_id, _project_cert_subject(project_id)) os.chdir(start) @@ -228,8 +226,8 @@ def generate_vpn_files(project_id): start = os.getcwd() os.chdir(ca_folder()) # TODO(vish): the shell scripts could all be done in python - utils.execute("sh genvpn.sh %s %s" % - (project_id, _vpn_cert_subject(project_id))) + utils.execute('sh','genvpn.sh', + project_id, _vpn_cert_subject(project_id)) with open(csr_fn, "r") as csrfile: csr_text = csrfile.read() (serial, signed_csr) = sign_csr(csr_text, project_id) @@ -259,9 +257,9 @@ def _sign_csr(csr_text, ca_folder): start = os.getcwd() # Change working dir to CA os.chdir(ca_folder) - utils.execute("openssl ca -batch -out %s -config " - "./openssl.cnf -infiles %s" % (outbound, inbound)) - out, _err = utils.execute("openssl x509 -in %s -serial -noout" % outbound) + utils.execute('openssl','ca','-batch','-out',outbound,'-config' + './openssl.cnf','-infiles',inbound) + out, _err = utils.execute('openssl','x509','-in',outbound','-serial','-noout') serial = out.rpartition("=")[2] os.chdir(start) with open(outbound, "r") as crtfile: -- cgit From 90abcdc7ae9e3f855dadb1ccc88892a2cc7bab05 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 27 Feb 2011 20:57:13 -0500 Subject: execvp --- nova/console/xvp.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/nova/console/xvp.py b/nova/console/xvp.py index cd257e0a6..271dffa54 100644 --- a/nova/console/xvp.py +++ b/nova/console/xvp.py @@ -133,10 +133,10 @@ class XVPConsoleProxy(object): return logging.debug(_("Starting xvp")) try: - utils.execute('xvp -p %s -c %s -l %s' % - (FLAGS.console_xvp_pid, - FLAGS.console_xvp_conf, - FLAGS.console_xvp_log)) + utils.execute('xvp', + '-p',FLAGS.console_xvp_pid, + '-c',FLAGS.console_xvp_conf, + '-l',FLAGS.console_xvp_log) except exception.ProcessExecutionError, err: logging.error(_("Error starting xvp: %s") % err) @@ -190,5 +190,5 @@ class XVPConsoleProxy(object): flag = '-x' #xvp will blow up on passwords that are too long (mdragon) password = password[:maxlen] - out, err = utils.execute('xvp %s' % flag, process_input=password) + out, err = utils.execute('xvp', flag, process_input=password) return out.strip() -- cgit From a62e603e8b1cedd89ca0c71f1cdc928d19c68a4d Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 3 Mar 2011 11:04:33 -0500 Subject: adding wsgi.Request class to add custom best_match; adding new class to wsgify decorators; replacing all references to webob.Request in non-test code to wsgi.Request --- nova/api/direct.py | 4 ++-- nova/api/ec2/__init__.py | 14 +++++------ nova/api/ec2/metadatarequesthandler.py | 2 +- nova/api/openstack/__init__.py | 4 ++-- nova/api/openstack/auth.py | 4 ++-- nova/api/openstack/common.py | 2 +- nova/api/openstack/faults.py | 2 +- nova/api/openstack/ratelimiting/__init__.py | 4 ++-- nova/wsgi.py | 37 ++++++++++++++++++++++------- 9 files changed, 47 insertions(+), 26 deletions(-) diff --git a/nova/api/direct.py b/nova/api/direct.py index 208b6d086..cd237f649 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -187,7 +187,7 @@ class ServiceWrapper(wsgi.Controller): def __init__(self, service_handle): self.service_handle = service_handle - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): arg_dict = req.environ['wsgiorg.routing_args'][1] action = arg_dict['action'] @@ -218,7 +218,7 @@ class Proxy(object): self.prefix = prefix def __do_request(self, path, context, **kwargs): - req = webob.Request.blank(path) + req = wsgi.Request.blank(path) req.method = 'POST' req.body = urllib.urlencode({'json': utils.dumps(kwargs)}) req.environ['openstack.context'] = context diff --git a/nova/api/ec2/__init__.py b/nova/api/ec2/__init__.py index 5adc2c075..58b1ecf03 100644 --- a/nova/api/ec2/__init__.py +++ b/nova/api/ec2/__init__.py @@ -53,7 +53,7 @@ flags.DEFINE_list('lockout_memcached_servers', None, class RequestLogging(wsgi.Middleware): """Access-Log akin logging for all EC2 API requests.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): start = utils.utcnow() rv = req.get_response(self.application) @@ -112,7 +112,7 @@ class Lockout(wsgi.Middleware): debug=0) super(Lockout, self).__init__(application) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): access_key = str(req.params['AWSAccessKeyId']) failures_key = "authfailures-%s" % access_key @@ -141,7 +141,7 @@ class Authenticate(wsgi.Middleware): """Authenticate an EC2 request and add 'ec2.context' to WSGI environ.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): # Read request signature and access id. try: @@ -190,7 +190,7 @@ class Requestify(wsgi.Middleware): super(Requestify, self).__init__(app) self.controller = utils.import_class(controller)() - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): non_args = ['Action', 'Signature', 'AWSAccessKeyId', 'SignatureMethod', 'SignatureVersion', 'Version', 'Timestamp'] @@ -269,7 +269,7 @@ class Authorizer(wsgi.Middleware): }, } - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): context = req.environ['ec2.context'] controller = req.environ['ec2.request'].controller.__class__.__name__ @@ -303,7 +303,7 @@ class Executor(wsgi.Application): response, or a 400 upon failure. """ - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): context = req.environ['ec2.context'] api_request = req.environ['ec2.request'] @@ -365,7 +365,7 @@ class Executor(wsgi.Application): class Versions(wsgi.Application): - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Respond to a request for all EC2 versions.""" # available api versions diff --git a/nova/api/ec2/metadatarequesthandler.py b/nova/api/ec2/metadatarequesthandler.py index 6fb441656..28f99b0ef 100644 --- a/nova/api/ec2/metadatarequesthandler.py +++ b/nova/api/ec2/metadatarequesthandler.py @@ -65,7 +65,7 @@ class MetadataRequestHandler(wsgi.Application): data = data[item] return data - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): cc = cloud.CloudController() remote_address = req.remote_addr diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 274330e3b..b5439788d 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -47,7 +47,7 @@ flags.DEFINE_bool('allow_admin_api', class FaultWrapper(wsgi.Middleware): """Calls down the middleware stack, making exceptions into faults.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): try: return req.get_response(self.application) @@ -115,7 +115,7 @@ class APIRouter(wsgi.Router): class Versions(wsgi.Application): - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Respond to a request for all OpenStack API versions.""" response = { diff --git a/nova/api/openstack/auth.py b/nova/api/openstack/auth.py index 6011e6115..de8905f46 100644 --- a/nova/api/openstack/auth.py +++ b/nova/api/openstack/auth.py @@ -46,7 +46,7 @@ class AuthMiddleware(wsgi.Middleware): self.auth = auth.manager.AuthManager() super(AuthMiddleware, self).__init__(application) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): if not self.has_authentication(req): return self.authenticate(req) @@ -121,7 +121,7 @@ class AuthMiddleware(wsgi.Middleware): username - string key - string API key - req - webob.Request object + req - wsgi.Request object """ ctxt = context.get_admin_context() user = self.auth.get_user_from_access_key(key) diff --git a/nova/api/openstack/common.py b/nova/api/openstack/common.py index 9f85c5c8a..49b970d75 100644 --- a/nova/api/openstack/common.py +++ b/nova/api/openstack/common.py @@ -25,7 +25,7 @@ def limited(items, request, max_limit=1000): Return a slice of items according to requested offset and limit. @param items: A sliceable entity - @param request: `webob.Request` possibly containing 'offset' and 'limit' + @param request: `wsgi.Request` possibly containing 'offset' and 'limit' GET variables. 'offset' is where to start in the list, and 'limit' is the maximum number of items to return. If 'limit' is not specified, 0, or > max_limit, we default diff --git a/nova/api/openstack/faults.py b/nova/api/openstack/faults.py index 224a7ef0b..c70b00fa3 100644 --- a/nova/api/openstack/faults.py +++ b/nova/api/openstack/faults.py @@ -42,7 +42,7 @@ class Fault(webob.exc.HTTPException): """Create a Fault for the given webob.exc.exception.""" self.wrapped_exc = exception - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Generate a WSGI response based on the exception passed to ctor.""" # Replace the body with fault details. diff --git a/nova/api/openstack/ratelimiting/__init__.py b/nova/api/openstack/ratelimiting/__init__.py index cbb4b897e..88ffc3246 100644 --- a/nova/api/openstack/ratelimiting/__init__.py +++ b/nova/api/openstack/ratelimiting/__init__.py @@ -57,7 +57,7 @@ class RateLimitingMiddleware(wsgi.Middleware): self.limiter = WSGIAppProxy(service_host) super(RateLimitingMiddleware, self).__init__(application) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): """Rate limit the request. @@ -183,7 +183,7 @@ class WSGIApp(object): """Create the WSGI application using the given Limiter instance.""" self.limiter = limiter - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=wsgi.Request) def __call__(self, req): parts = req.path_info.split('/') # format: /limiter// diff --git a/nova/wsgi.py b/nova/wsgi.py index 1eb66d067..67216d540 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -82,6 +82,27 @@ class Server(object): log=WritableLogger(logger)) +class Request(webob.Request): + + def best_match(self): + """ + Determine the most acceptable content-type based on the + query extension then the Accept header + """ + + parts = self.path.rsplit(".", 1) + + if len(parts) > 1: + format = parts[1] + if format in ["json", "xml"]: + return "application/{0}".format(parts[1]) + + ctypes = ["application/json", "application/xml"] + bm = self.accept.best_match(ctypes) + + return bm or "application/json" + + class Application(object): """Base WSGI application wrapper. Subclasses need to implement __call__.""" @@ -113,7 +134,7 @@ class Application(object): def __call__(self, environ, start_response): r"""Subclasses will probably want to implement __call__ like this: - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): # Any of the following objects work as responses: @@ -199,7 +220,7 @@ class Middleware(Application): """Do whatever you'd like to the response.""" return response - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): response = self.process_request(req) if response: @@ -212,7 +233,7 @@ class Debug(Middleware): """Helper class that can be inserted into any WSGI application chain to get information about the request and response.""" - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): print ("*" * 40) + " REQUEST ENVIRON" for key, value in req.environ.items(): @@ -276,7 +297,7 @@ class Router(object): self._router = routes.middleware.RoutesMiddleware(self._dispatch, self.map) - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): """ Route the incoming request to a controller based on self.map. @@ -285,7 +306,7 @@ class Router(object): return self._router @staticmethod - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def _dispatch(req): """ Called by self._router after matching the incoming request to a route @@ -304,11 +325,11 @@ class Controller(object): WSGI app that reads routing information supplied by RoutesMiddleware and calls the requested action method upon itself. All action methods must, in addition to their normal parameters, accept a 'req' argument - which is the incoming webob.Request. They raise a webob.exc exception, + which is the incoming wsgi.Request. They raise a webob.exc exception, or return a dict which will be serialized by requested content type. """ - @webob.dec.wsgify + @webob.dec.wsgify(RequestClass=Request) def __call__(self, req): """ Call the method specified in req.environ by RoutesMiddleware. @@ -358,7 +379,7 @@ class Serializer(object): needed to serialize a dictionary to that type. """ self.metadata = metadata or {} - req = webob.Request.blank('', environ) + req = wsgi.Request.blank('', environ) suffix = req.path_info.split('.')[-1].lower() if suffix == 'json': self.handler = self._to_json -- cgit From 6d075754bdd4090342bf4f79c726a52923c311a8 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 3 Mar 2011 12:45:34 -0500 Subject: adding wsgi.Controller and wsgi.Request testing; fixing format keyword argument exception --- nova/tests/api/test_wsgi.py | 120 +++++++++++++++++++++++++++++++++++++------- nova/wsgi.py | 4 +- 2 files changed, 105 insertions(+), 19 deletions(-) diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index 2c7852214..cf2d0e297 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -21,11 +21,13 @@ Test WSGI basics and provide some helper functions for other WSGI tests. """ +import json from nova import test import routes import webob +from nova import exception from nova import wsgi @@ -66,30 +68,112 @@ class Test(test.TestCase): result = webob.Request.blank('/bad').get_response(Router()) self.assertNotEqual(result.body, "Router result") - def test_controller(self): - class Controller(wsgi.Controller): - """Test controller to call from router.""" - test = self +class ControllerTest(test.TestCase): + + class TestRouter(wsgi.Router): + + class TestController(wsgi.Controller): + + _serialization_metadata = { + 'application/xml': { + "attributes": { + "test": ["id"]}}} def show(self, req, id): # pylint: disable-msg=W0622,C0103 - """Default action called for requests with an ID.""" - self.test.assertEqual(req.path_info, '/tests/123') - self.test.assertEqual(id, '123') - return id + return {"test": {"id": id}} + + def __init__(self): + mapper = routes.Mapper() + mapper.resource("test", "tests", controller=self.TestController()) + wsgi.Router.__init__(self, mapper) + + def test_show(self): + request = wsgi.Request.blank('/tests/123') + result = request.get_response(self.TestRouter()) + self.assertEqual(json.loads(result.body), {"test": {"id": "123"}}) + + def test_content_type_from_accept_xml(self): + request = webob.Request.blank('/tests/123') + request.headers["Accept"] = "application/xml" + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/xml") + + def test_content_type_from_accept_json(self): + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/json" + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/json") + + def test_content_type_from_query_extension_xml(self): + request = wsgi.Request.blank('/tests/123.xml') + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/xml") + + def test_content_type_from_query_extension_json(self): + request = wsgi.Request.blank('/tests/123.json') + result = request.get_response(self.TestRouter()) + self.assertEqual(result.headers["Content-Type"], "application/json") + + def test_content_type_default_when_unsupported(self): + request = wsgi.Request.blank('/tests/123.unsupported') + request.headers["Accept"] = "application/unsupported1" + result = request.get_response(self.TestRouter()) + self.assertEqual(result.status_int, 200) + self.assertEqual(result.headers["Content-Type"], "application/json") + + +class RequestTest(test.TestCase): + + def test_content_type_from_accept_xml(self): + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/xml" + result = request.best_match() + self.assertEqual(result, "application/xml") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/json" + result = request.best_match() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = "application/xml, application/json" + result = request.best_match() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123') + request.headers["Accept"] = \ + "application/json; q=0.3, application/xml; q=0.9" + result = request.best_match() + self.assertEqual(result, "application/xml") + + def test_content_type_from_query_extension(self): + request = wsgi.Request.blank('/tests/123.xml') + result = request.best_match() + self.assertEqual(result, "application/xml") + + request = wsgi.Request.blank('/tests/123.json') + result = request.best_match() + self.assertEqual(result, "application/json") + + request = wsgi.Request.blank('/tests/123.invalid') + result = request.best_match() + self.assertEqual(result, "application/json") + + def test_content_type_accept_and_query_extension(self): + request = wsgi.Request.blank('/tests/123.xml') + request.headers["Accept"] = "application/json" + result = request.best_match() + self.assertEqual(result, "application/xml") + + def test_content_type_accept_default(self): + request = wsgi.Request.blank('/tests/123.unsupported') + request.headers["Accept"] = "application/unsupported1" + result = request.best_match() + self.assertEqual(result, "application/json") - class Router(wsgi.Router): - """Test router.""" - def __init__(self): - mapper = routes.Mapper() - mapper.resource("test", "tests", controller=Controller()) - super(Router, self).__init__(mapper) - result = webob.Request.blank('/tests/123').get_response(Router()) - self.assertEqual(result.body, "123") - result = webob.Request.blank('/test/123').get_response(Router()) - self.assertNotEqual(result.body, "123") class SerializerTest(test.TestCase): diff --git a/nova/wsgi.py b/nova/wsgi.py index 67216d540..4577439cb 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -339,6 +339,8 @@ class Controller(object): method = getattr(self, action) del arg_dict['controller'] del arg_dict['action'] + if 'format' in arg_dict: + del arg_dict['format'] arg_dict['req'] = req result = method(**arg_dict) if type(result) is dict: @@ -379,7 +381,7 @@ class Serializer(object): needed to serialize a dictionary to that type. """ self.metadata = metadata or {} - req = wsgi.Request.blank('', environ) + req = Request.blank('', environ) suffix = req.path_info.split('.')[-1].lower() if suffix == 'json': self.handler = self._to_json -- cgit From 848aced747a60c47d76efcb2147041339df4a628 Mon Sep 17 00:00:00 2001 From: Brian Waldon Date: Thu, 3 Mar 2011 17:21:21 -0500 Subject: Refactor wsgi.Serializer away from handling Requests directly; now require Content-Type in all requests; fix tests according to new code --- nova/api/direct.py | 2 +- nova/api/openstack/__init__.py | 4 +- nova/api/openstack/consoles.py | 2 +- nova/api/openstack/faults.py | 5 +- nova/api/openstack/images.py | 2 +- nova/api/openstack/servers.py | 9 ++- nova/api/openstack/zones.py | 4 +- nova/exception.py | 4 ++ nova/tests/api/openstack/test_servers.py | 1 + nova/tests/api/openstack/test_zones.py | 15 +++-- nova/tests/api/test_wsgi.py | 95 +++++++++++++++++++------------ nova/tests/test_direct.py | 3 + nova/wsgi.py | 96 ++++++++++++++++++++------------ 13 files changed, 152 insertions(+), 90 deletions(-) diff --git a/nova/api/direct.py b/nova/api/direct.py index cd237f649..1d699f947 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -206,7 +206,7 @@ class ServiceWrapper(wsgi.Controller): params = dict([(str(k), v) for (k, v) in params.iteritems()]) result = method(context, **params) if type(result) is dict or type(result) is list: - return self._serialize(result, req) + return self._serialize(result, req.best_match()) else: return result diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index b5439788d..6e1a2a06c 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -124,4 +124,6 @@ class Versions(wsgi.Application): metadata = { "application/xml": { "attributes": dict(version=["status", "id"])}} - return wsgi.Serializer(req.environ, metadata).to_content_type(response) + + content_type = req.best_match() + return wsgi.Serializer(metadata).serialize(response, content_type) diff --git a/nova/api/openstack/consoles.py b/nova/api/openstack/consoles.py index 9ebdbe710..8c291c2eb 100644 --- a/nova/api/openstack/consoles.py +++ b/nova/api/openstack/consoles.py @@ -65,7 +65,7 @@ class Controller(wsgi.Controller): def create(self, req, server_id): """Creates a new console""" - #info = self._deserialize(req.body, req) + #info = self._deserialize(req.body, req.get_content_type()) self.console_api.create_console( req.environ['nova.context'], int(server_id)) diff --git a/nova/api/openstack/faults.py b/nova/api/openstack/faults.py index c70b00fa3..075fdb997 100644 --- a/nova/api/openstack/faults.py +++ b/nova/api/openstack/faults.py @@ -57,6 +57,7 @@ class Fault(webob.exc.HTTPException): fault_data[fault_name]['retryAfter'] = retry # 'code' is an attribute on the fault tag itself metadata = {'application/xml': {'attributes': {fault_name: 'code'}}} - serializer = wsgi.Serializer(req.environ, metadata) - self.wrapped_exc.body = serializer.to_content_type(fault_data) + serializer = wsgi.Serializer(metadata) + content_type = req.best_match() + self.wrapped_exc.body = serializer.serialize(fault_data, content_type) return self.wrapped_exc diff --git a/nova/api/openstack/images.py b/nova/api/openstack/images.py index cf85a496f..98f0dd96b 100644 --- a/nova/api/openstack/images.py +++ b/nova/api/openstack/images.py @@ -151,7 +151,7 @@ class Controller(wsgi.Controller): def create(self, req): context = req.environ['nova.context'] - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) instance_id = env["image"]["serverId"] name = env["image"]["name"] diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 08b95b46a..24d2826af 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -141,7 +141,7 @@ class Controller(wsgi.Controller): def create(self, req): """ Creates a new server for a given user """ - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) if not env: return faults.Fault(exc.HTTPUnprocessableEntity()) @@ -182,7 +182,10 @@ class Controller(wsgi.Controller): def update(self, req, id): """ Updates the server name or password """ - inst_dict = self._deserialize(req.body, req) + if len(req.body) == 0: + raise exc.HTTPUnprocessableEntity() + + inst_dict = self._deserialize(req.body, req.get_content_type()) if not inst_dict: return faults.Fault(exc.HTTPUnprocessableEntity()) @@ -205,7 +208,7 @@ class Controller(wsgi.Controller): def action(self, req, id): """ Multi-purpose method used to reboot, rebuild, and resize a server """ - input_dict = self._deserialize(req.body, req) + input_dict = self._deserialize(req.body, req.get_content_type()) #TODO(sandy): rebuild/resize not supported. try: reboot_type = input_dict['reboot']['type'] diff --git a/nova/api/openstack/zones.py b/nova/api/openstack/zones.py index d5206da20..cf6cd789f 100644 --- a/nova/api/openstack/zones.py +++ b/nova/api/openstack/zones.py @@ -67,13 +67,13 @@ class Controller(wsgi.Controller): def create(self, req): context = req.environ['nova.context'] - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) zone = db.zone_create(context, env["zone"]) return dict(zone=_scrub_zone(zone)) def update(self, req, id): context = req.environ['nova.context'] - env = self._deserialize(req.body, req) + env = self._deserialize(req.body, req.get_content_type()) zone_id = int(id) zone = db.zone_update(context, zone_id, env["zone"]) return dict(zone=_scrub_zone(zone)) diff --git a/nova/exception.py b/nova/exception.py index 7d65bd6a5..93c5fe3d7 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -88,6 +88,10 @@ class InvalidInputException(Error): pass +class InvalidContentType(Error): + pass + + class TimeoutException(Error): pass diff --git a/nova/tests/api/openstack/test_servers.py b/nova/tests/api/openstack/test_servers.py index 78beb7df9..fae08d0be 100644 --- a/nova/tests/api/openstack/test_servers.py +++ b/nova/tests/api/openstack/test_servers.py @@ -227,6 +227,7 @@ class ServersTest(test.TestCase): req = webob.Request.blank('/v1.0/servers') req.method = 'POST' req.body = json.dumps(body) + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) diff --git a/nova/tests/api/openstack/test_zones.py b/nova/tests/api/openstack/test_zones.py index 555b206b9..d0da8eaaf 100644 --- a/nova/tests/api/openstack/test_zones.py +++ b/nova/tests/api/openstack/test_zones.py @@ -86,24 +86,27 @@ class ZonesTest(test.TestCase): def test_get_zone_list(self): req = webob.Request.blank('/v1.0/zones') + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(len(res_dict['zones']), 2) def test_get_zone_by_id(self): req = webob.Request.blank('/v1.0/zones/1') + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) + self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://foo.com') self.assertFalse('password' in res_dict['zone']) - self.assertEqual(res.status_int, 200) def test_zone_delete(self): req = webob.Request.blank('/v1.0/zones/1') + req.headers["Content-Type"] = "application/json" res = req.get_response(fakes.wsgi_app()) self.assertEqual(res.status_int, 200) @@ -112,13 +115,14 @@ class ZonesTest(test.TestCase): body = dict(zone=dict(api_url='http://blah.zoo', username='fred', password='fubar')) req = webob.Request.blank('/v1.0/zones') + req.headers["Content-Type"] = "application/json" req.method = 'POST' req.body = json.dumps(body) res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://blah.zoo') self.assertFalse('username' in res_dict['zone']) @@ -126,13 +130,14 @@ class ZonesTest(test.TestCase): def test_zone_update(self): body = dict(zone=dict(username='zeb', password='sneaky')) req = webob.Request.blank('/v1.0/zones/1') + req.headers["Content-Type"] = "application/json" req.method = 'PUT' req.body = json.dumps(body) res = req.get_response(fakes.wsgi_app()) - res_dict = json.loads(res.body) self.assertEqual(res.status_int, 200) + res_dict = json.loads(res.body) self.assertEqual(res_dict['zone']['id'], 1) self.assertEqual(res_dict['zone']['api_url'], 'http://foo.com') self.assertFalse('username' in res_dict['zone']) diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index cf2d0e297..7c0135656 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -93,29 +93,29 @@ class ControllerTest(test.TestCase): result = request.get_response(self.TestRouter()) self.assertEqual(json.loads(result.body), {"test": {"id": "123"}}) - def test_content_type_from_accept_xml(self): + def test_response_content_type_from_accept_xml(self): request = webob.Request.blank('/tests/123') request.headers["Accept"] = "application/xml" result = request.get_response(self.TestRouter()) self.assertEqual(result.headers["Content-Type"], "application/xml") - def test_content_type_from_accept_json(self): + def test_response_content_type_from_accept_json(self): request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/json" result = request.get_response(self.TestRouter()) self.assertEqual(result.headers["Content-Type"], "application/json") - def test_content_type_from_query_extension_xml(self): + def test_response_content_type_from_query_extension_xml(self): request = wsgi.Request.blank('/tests/123.xml') result = request.get_response(self.TestRouter()) self.assertEqual(result.headers["Content-Type"], "application/xml") - def test_content_type_from_query_extension_json(self): + def test_response_content_type_from_query_extension_json(self): request = wsgi.Request.blank('/tests/123.json') result = request.get_response(self.TestRouter()) self.assertEqual(result.headers["Content-Type"], "application/json") - def test_content_type_default_when_unsupported(self): + def test_response_content_type_default_when_unsupported(self): request = wsgi.Request.blank('/tests/123.unsupported') request.headers["Accept"] = "application/unsupported1" result = request.get_response(self.TestRouter()) @@ -125,6 +125,17 @@ class ControllerTest(test.TestCase): class RequestTest(test.TestCase): + def test_request_content_type_missing(self): + request = wsgi.Request.blank('/tests/123') + request.body = "" + self.assertRaises(webob.exc.HTTPBadRequest, request.get_content_type) + + def test_request_content_type_unsupported(self): + request = wsgi.Request.blank('/tests/123') + request.headers["Content-Type"] = "text/html" + request.body = "asdf
" + self.assertRaises(webob.exc.HTTPBadRequest, request.get_content_type) + def test_content_type_from_accept_xml(self): request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/xml" @@ -173,40 +184,48 @@ class RequestTest(test.TestCase): self.assertEqual(result, "application/json") - - - class SerializerTest(test.TestCase): - def match(self, url, accept, expect): + def test_xml(self): input_dict = dict(servers=dict(a=(2, 3))) expected_xml = '(2,3)' + serializer = wsgi.Serializer() + result = serializer.serialize(input_dict, "application/xml") + result = result.replace('\n', '').replace(' ', '') + self.assertEqual(result, expected_xml) + + def test_json(self): + input_dict = dict(servers=dict(a=(2, 3))) expected_json = '{"servers":{"a":[2,3]}}' - req = webob.Request.blank(url, headers=dict(Accept=accept)) - result = wsgi.Serializer(req.environ).to_content_type(input_dict) + serializer = wsgi.Serializer() + result = serializer.serialize(input_dict, "application/json") result = result.replace('\n', '').replace(' ', '') - if expect == 'xml': - self.assertEqual(result, expected_xml) - elif expect == 'json': - self.assertEqual(result, expected_json) - else: - raise "Bad expect value" - - def test_basic(self): - self.match('/servers/4.json', None, expect='json') - self.match('/servers/4', 'application/json', expect='json') - self.match('/servers/4', 'application/xml', expect='xml') - self.match('/servers/4.xml', None, expect='xml') - - def test_defaults_to_json(self): - self.match('/servers/4', None, expect='json') - self.match('/servers/4', 'text/html', expect='json') - - def test_suffix_takes_precedence_over_accept_header(self): - self.match('/servers/4.xml', 'application/json', expect='xml') - self.match('/servers/4.xml.', 'application/json', expect='json') - - def test_deserialize(self): + self.assertEqual(result, expected_json) + + def test_unsupported_content_type(self): + serializer = wsgi.Serializer() + self.assertRaises(exception.InvalidContentType, serializer.serialize, + {}, "text/null") + + def test_deserialize_json(self): + data = """{"a": { + "a1": "1", + "a2": "2", + "bs": ["1", "2", "3", {"c": {"c1": "1"}}], + "d": {"e": "1"}, + "f": "1"}}""" + as_dict = dict(a={ + 'a1': '1', + 'a2': '2', + 'bs': ['1', '2', '3', {'c': dict(c1='1')}], + 'd': {'e': '1'}, + 'f': '1'}) + metadata = {} + serializer = wsgi.Serializer(metadata) + self.assertEqual(serializer.deserialize(data, "application/json"), + as_dict) + + def test_deserialize_xml(self): xml = """ 123 @@ -221,11 +240,13 @@ class SerializerTest(test.TestCase): 'd': {'e': '1'}, 'f': '1'}) metadata = {'application/xml': dict(plurals={'bs': 'b', 'ts': 't'})} - serializer = wsgi.Serializer({}, metadata) - self.assertEqual(serializer.deserialize(xml), as_dict) + serializer = wsgi.Serializer(metadata) + self.assertEqual(serializer.deserialize(xml, "application/xml"), + as_dict) def test_deserialize_empty_xml(self): xml = """""" as_dict = {"a": {}} - serializer = wsgi.Serializer({}) - self.assertEqual(serializer.deserialize(xml), as_dict) + serializer = wsgi.Serializer() + self.assertEqual(serializer.deserialize(xml, "application/xml"), + as_dict) diff --git a/nova/tests/test_direct.py b/nova/tests/test_direct.py index b6bfab534..85bfcfd85 100644 --- a/nova/tests/test_direct.py +++ b/nova/tests/test_direct.py @@ -59,6 +59,7 @@ class DirectTestCase(test.TestCase): req.headers['X-OpenStack-User'] = 'user1' req.headers['X-OpenStack-Project'] = 'proj1' resp = req.get_response(self.auth_router) + self.assertEqual(resp.status_int, 200) data = json.loads(resp.body) self.assertEqual(data['user'], 'user1') self.assertEqual(data['project'], 'proj1') @@ -69,6 +70,7 @@ class DirectTestCase(test.TestCase): req.method = 'POST' req.body = 'json=%s' % json.dumps({'data': 'foo'}) resp = req.get_response(self.router) + self.assertEqual(resp.status_int, 200) resp_parsed = json.loads(resp.body) self.assertEqual(resp_parsed['data'], 'foo') @@ -78,6 +80,7 @@ class DirectTestCase(test.TestCase): req.method = 'POST' req.body = 'data=foo' resp = req.get_response(self.router) + self.assertEqual(resp.status_int, 200) resp_parsed = json.loads(resp.body) self.assertEqual(resp_parsed['data'], 'foo') diff --git a/nova/wsgi.py b/nova/wsgi.py index 4577439cb..c3e08522d 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -36,6 +36,7 @@ import webob.exc from paste import deploy +from nova import exception from nova import flags from nova import log as logging from nova import utils @@ -102,6 +103,14 @@ class Request(webob.Request): return bm or "application/json" + def get_content_type(self): + try: + ct = self.headers["Content-Type"] + assert ct in ("application/xml", "application/json") + return ct + except Exception: + raise webob.exc.HTTPBadRequest("Invalid content type") + class Application(object): """Base WSGI application wrapper. Subclasses need to implement __call__.""" @@ -343,30 +352,41 @@ class Controller(object): del arg_dict['format'] arg_dict['req'] = req result = method(**arg_dict) + if type(result) is dict: - return self._serialize(result, req) + content_type = req.best_match() + body = self._serialize(result, content_type) + + response = webob.Response() + response.headers["Content-Type"] = content_type + response.body = body + return response + else: return result - def _serialize(self, data, request): + def _serialize(self, data, content_type): """ - Serialize the given dict to the response type requested in request. + Serialize the given dict to the provided content_type. Uses self._serialization_metadata if it exists, which is a dict mapping MIME types to information needed to serialize to that type. """ _metadata = getattr(type(self), "_serialization_metadata", {}) - serializer = Serializer(request.environ, _metadata) - return serializer.to_content_type(data) + serializer = Serializer(_metadata) + try: + return serializer.serialize(data, content_type) + except exception.InvalidContentType: + raise webob.exc.HTTPNotAcceptable() - def _deserialize(self, data, request): + def _deserialize(self, data, content_type): """ - Deserialize the request body to the response type requested in request. + Deserialize the request body to the specefied content type. Uses self._serialization_metadata if it exists, which is a dict mapping MIME types to information needed to serialize to that type. """ _metadata = getattr(type(self), "_serialization_metadata", {}) - serializer = Serializer(request.environ, _metadata) - return serializer.deserialize(data) + serializer = Serializer(_metadata) + return serializer.deserialize(data, content_type) class Serializer(object): @@ -374,50 +394,52 @@ class Serializer(object): Serializes and deserializes dictionaries to certain MIME types. """ - def __init__(self, environ, metadata=None): + def __init__(self, metadata=None): """ Create a serializer based on the given WSGI environment. 'metadata' is an optional dict mapping MIME types to information needed to serialize a dictionary to that type. """ self.metadata = metadata or {} - req = Request.blank('', environ) - suffix = req.path_info.split('.')[-1].lower() - if suffix == 'json': - self.handler = self._to_json - elif suffix == 'xml': - self.handler = self._to_xml - elif 'application/json' in req.accept: - self.handler = self._to_json - elif 'application/xml' in req.accept: - self.handler = self._to_xml - else: - # This is the default - self.handler = self._to_json - def to_content_type(self, data): - """ - Serialize a dictionary into a string. + def _get_serialize_handler(self, content_type): + handlers = { + "application/json": self._to_json, + "application/xml": self._to_xml, + } + + try: + return handlers[content_type] + except Exception: + raise exception.InvalidContentType() - The format of the string will be decided based on the Content Type - requested in self.environ: by Accept: header, or by URL suffix. + def serialize(self, data, content_type): + """ + Serialize a dictionary into a string of the specified content type. """ - return self.handler(data) + return self._get_serialize_handler(content_type)(data) - def deserialize(self, datastring): + def deserialize(self, datastring, content_type): """ Deserialize a string to a dictionary. The string must be in the format of a supported MIME type. """ - datastring = datastring.strip() + return self.get_deserialize_handler(content_type)(datastring) + + def get_deserialize_handler(self, content_type): + handlers = { + "application/json": self._from_json, + "application/xml": self._from_xml, + } + try: - is_xml = (datastring[0] == '<') - if not is_xml: - return utils.loads(datastring) - return self._from_xml(datastring) - except: - return None + return handlers[content_type] + except Exception: + raise exception.InvalidContentType() + + def _from_json(self, datastring): + return utils.loads(datastring) def _from_xml(self, datastring): xmldata = self.metadata.get('application/xml', {}) -- cgit From cac5881eaa35f94e004c18dd34ca78014f067976 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Tue, 8 Mar 2011 01:01:41 -0500 Subject: execvp --- nova/crypto.py | 32 +-- nova/network/linux_net.py | 222 +++++++++++---------- nova/tests/test_network.py | 16 +- nova/utils.py | 16 +- nova/virt/disk.py | 44 ++-- nova/virt/images.py | 5 +- nova/virt/libvirt_conn.py | 36 ++-- nova/virt/xenapi/vm_utils.py | 11 +- nova/volume/driver.py | 71 +++---- .../networking/etc/xensource/scripts/vif_rules.py | 91 ++++++--- 10 files changed, 296 insertions(+), 248 deletions(-) diff --git a/nova/crypto.py b/nova/crypto.py index b240a3958..dd24723b8 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -105,8 +105,10 @@ def generate_key_pair(bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.join(tmpdir, 'temp') - utils.execute('ssh-keygen','-q','-b',"%d" % bits,'-N','""','-f',keyfile) - (out, err) = utils.execute('ssh-keygen','-q','-l','-f',"%s.pub" % (keyfile)) + utils.execute('ssh-keygen', '-q', '-b', '%d' % bits, '-N', '', + '-f', keyfile) + (out, err) = utils.execute('ssh-keygen', '-q', '-l', '-f', + '%s.pub' % (keyfile)) fingerprint = out.split(' ')[1] private_key = open(keyfile).read() public_key = open(keyfile + '.pub').read() @@ -118,7 +120,7 @@ def generate_key_pair(bits=1024): # bio = M2Crypto.BIO.MemoryBuffer() # key.save_pub_key_bio(bio) # public_key = bio.read() - # public_key, err = execute('ssh-keygen','-y','-f','/dev/stdin', private_key) + # public_key, err = execute('ssh-keygen', '-y', '-f', '/dev/stdin', private_key) return (private_key, public_key, fingerprint) @@ -143,9 +145,10 @@ def revoke_cert(project_id, file_name): start = os.getcwd() os.chdir(ca_folder(project_id)) # NOTE(vish): potential race condition here - utils.execute('openssl','ca','-config','./openssl.cnf','-revoke',"'%s'" % file_name) - utils.execute('openssl','ca','-gencrl','-config','./openssl.cnf','-out',"'%s'" % - FLAGS.crl_file) + utils.execute('openssl', 'ca', '-config', './openssl.cnf', '-revoke', + '%s' % file_name) + utils.execute('openssl', 'ca', '-gencrl', '-config', './openssl.cnf', + '-out', '%s' % FLAGS.crl_file) os.chdir(start) @@ -193,8 +196,9 @@ def generate_x509_cert(user_id, project_id, bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key')) csrfile = os.path.join(tmpdir, 'temp.csr') - utils.execute('openssl','genrsa','-out',keyfile,bits) - utils.execute('openssl','req','-new','-key',keyfile,'-out',csrfile,'-batch','-subj',subject) + utils.execute('openssl', 'genrsa', '-out', keyfile, bits) + utils.execute('openssl', 'req', '-new', '-key', keyfile, '-out', csrfile, + '-batch', '-subj', subject) private_key = open(keyfile).read() csr = open(csrfile).read() shutil.rmtree(tmpdir) @@ -211,7 +215,8 @@ def _ensure_project_folder(project_id): if not os.path.exists(ca_path(project_id)): start = os.getcwd() os.chdir(ca_folder()) - utils.execute('sh','geninter.sh',project_id, _project_cert_subject(project_id)) + utils.execute('sh', 'geninter.sh', project_id, + _project_cert_subject(project_id)) os.chdir(start) @@ -226,7 +231,7 @@ def generate_vpn_files(project_id): start = os.getcwd() os.chdir(ca_folder()) # TODO(vish): the shell scripts could all be done in python - utils.execute('sh','genvpn.sh', + utils.execute('sh', 'genvpn.sh', project_id, _vpn_cert_subject(project_id)) with open(csr_fn, "r") as csrfile: csr_text = csrfile.read() @@ -257,9 +262,10 @@ def _sign_csr(csr_text, ca_folder): start = os.getcwd() # Change working dir to CA os.chdir(ca_folder) - utils.execute('openssl','ca','-batch','-out',outbound,'-config' - './openssl.cnf','-infiles',inbound) - out, _err = utils.execute('openssl','x509','-in',outbound','-serial','-noout') + utils.execute('openssl', 'ca', '-batch', '-out', outbound, '-config', + './openssl.cnf', '-infiles', inbound) + out, _err = utils.execute('openssl', 'x509', '-in', outbound, + '-serial', '-noout') serial = out.rpartition("=")[2] os.chdir(start) with open(outbound, "r") as crtfile: diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index 535ce87bc..ad019a8c0 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -65,113 +65,117 @@ flags.DEFINE_string('dmz_cidr', '10.128.0.0/24', def metadata_forward(): """Create forwarding rule for metadata""" - _confirm_rule("PREROUTING", "-t nat -s 0.0.0.0/0 " - "-d 169.254.169.254/32 -p tcp -m tcp --dport 80 -j DNAT " - "--to-destination %s:%s" % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) + _confirm_rule("PREROUTING", '-t', 'nat', '-s', '0.0.0.0/0', + '-d', '169.254.169.254/32', '-p', 'tcp', '-m', 'tcp', + '--dport', '80', '-j', 'DNAT', + '--to-destination', '%s:%s' % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) def init_host(): """Basic networking setup goes here""" if FLAGS.use_nova_chains: - _execute("sudo iptables -N nova_input", check_exit_code=False) - _execute("sudo iptables -D %s -j nova_input" % FLAGS.input_chain, + _execute('sudo', 'iptables', '-N', 'nova_input', check_exit_code=False) + _execute('sudo', 'iptables', '-D', FLAGS.input_chain, + '-j', 'nova_input', check_exit_code=False) - _execute("sudo iptables -A %s -j nova_input" % FLAGS.input_chain) - - _execute("sudo iptables -N nova_forward", check_exit_code=False) - _execute("sudo iptables -D FORWARD -j nova_forward", + _execute('sudo', 'iptables', '-A', FLAGS.input_chain, + '-j', 'nova_input') + _execute('sudo', 'iptables', '-N', 'nova_forward', check_exit_code=False) - _execute("sudo iptables -A FORWARD -j nova_forward") - - _execute("sudo iptables -N nova_output", check_exit_code=False) - _execute("sudo iptables -D OUTPUT -j nova_output", + _execute('sudo', 'iptables', '-D', 'FORWARD', '-j', 'nova_forward', check_exit_code=False) - _execute("sudo iptables -A OUTPUT -j nova_output") - - _execute("sudo iptables -t nat -N nova_prerouting", + _execute('sudo', 'iptables', '-A', 'FORWARD', '-j', 'nova_forward') + _execute('sudo', 'iptables', '-N', 'nova_output', check_exit_code=False) + _execute('sudo', 'iptables', '-D', 'OUTPUT', '-j', 'nova_output', check_exit_code=False) - _execute("sudo iptables -t nat -D PREROUTING -j nova_prerouting", + _execute('sudo', 'iptables', '-A', 'OUTPUT', '-j', 'nova_output') + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'nova_prerouting', check_exit_code=False) - _execute("sudo iptables -t nat -A PREROUTING -j nova_prerouting") - - _execute("sudo iptables -t nat -N nova_postrouting", + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'PREROUTING', + '-j', 'nova_prerouting', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'PREROUTING', + '-j', 'nova_prerouting') + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'nova_postrouting', check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j nova_postrouting", + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'POSTROUTING', + '-j', 'nova_postrouting', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'POSTROUTING', + '-j', 'nova_postrouting') + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'nova_snatting', check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j nova_postrouting") - - _execute("sudo iptables -t nat -N nova_snatting", + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'POSTROUTING', + '-j nova_snatting', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'POSTROUTING', + '-j', 'nova_snatting') + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'nova_output', check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j nova_snatting", - check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j nova_snatting") - - _execute("sudo iptables -t nat -N nova_output", check_exit_code=False) - _execute("sudo iptables -t nat -D OUTPUT -j nova_output", - check_exit_code=False) - _execute("sudo iptables -t nat -A OUTPUT -j nova_output") + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'OUTPUT', + '-j nova_output', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'OUTPUT', + '-j', 'nova_output') else: # NOTE(vish): This makes it easy to ensure snatting rules always # come after the accept rules in the postrouting chain - _execute("sudo iptables -t nat -N SNATTING", - check_exit_code=False) - _execute("sudo iptables -t nat -D POSTROUTING -j SNATTING", + _execute('sudo', 'iptables', '-t', 'nat', '-N', 'SNATTING', check_exit_code=False) - _execute("sudo iptables -t nat -A POSTROUTING -j SNATTING") + _execute('sudo', 'iptables', '-t', 'nat', '-D', 'POSTROUTING', + '-j', 'SNATTING', check_exit_code=False) + _execute('sudo', 'iptables', '-t', 'nat', '-A', 'POSTROUTING', + '-j', 'SNATTING') # NOTE(devcamcar): Cloud public SNAT entries and the default # SNAT rule for outbound traffic. - _confirm_rule("SNATTING", "-t nat -s %s " - "-j SNAT --to-source %s" - % (FLAGS.fixed_range, FLAGS.routing_source_ip), append=True) + _confirm_rule("SNATTING", '-t', 'nat', '-s', FLAGS.fixed_range, + '-j', 'SNAT', '--to-source', FLAGS.routing_source_ip, + append=True) - _confirm_rule("POSTROUTING", "-t nat -s %s -d %s -j ACCEPT" % - (FLAGS.fixed_range, FLAGS.dmz_cidr)) - _confirm_rule("POSTROUTING", "-t nat -s %(range)s -d %(range)s -j ACCEPT" % - {'range': FLAGS.fixed_range}) + _confirm_rule("POSTROUTING", '-t', 'nat', '-s', FLAGS.fixed_range, + '-d', FLAGS.dmz_cidr, '-j', 'ACCEPT') + _confirm_rule("POSTROUTING", '-t', 'nat', '-s', FLAGS.fixed_range, + '-d', FLAGS.fixed_range, '-j', 'ACCEPT') def bind_floating_ip(floating_ip, check_exit_code=True): """Bind ip to public interface""" - _execute("sudo ip addr add %s dev %s" % (floating_ip, - FLAGS.public_interface), + _execute('sudo', 'ip', 'addr', 'add', floating_ip, + 'dev', FLAGS.public_interface), check_exit_code=check_exit_code) def unbind_floating_ip(floating_ip): """Unbind a public ip from public interface""" - _execute("sudo ip addr del %s dev %s" % (floating_ip, - FLAGS.public_interface)) + _execute('sudo', 'ip', 'addr', 'del', floating_ip, + 'dev', FLAGS.public_interface)) def ensure_vlan_forward(public_ip, port, private_ip): """Sets up forwarding rules for vlan""" - _confirm_rule("FORWARD", "-d %s -p udp --dport 1194 -j ACCEPT" % - private_ip) - _confirm_rule("PREROUTING", - "-t nat -d %s -p udp --dport %s -j DNAT --to %s:1194" - % (public_ip, port, private_ip)) + _confirm_rule("FORWARD", '-d', private_ip, '-p', 'udp', + '--dport', '1194', '-j', 'ACCEPT') + _confirm_rule("PREROUTING", '-t', 'nat', '-d', public_ip, '-p', 'udp', + '--dport', port, '-j', 'DNAT', '--to', '%s:1194' + % private_ip) def ensure_floating_forward(floating_ip, fixed_ip): """Ensure floating ip forwarding rule""" - _confirm_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _confirm_rule("OUTPUT", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _confirm_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" - % (fixed_ip, floating_ip)) + _confirm_rule("PREROUTING", '-t', 'nat', '-d', floating_ip, '-j', 'DNAT', + '--to', fixed_ip) + _confirm_rule("OUTPUT", '-t', 'nat', '-d', floating_ip, '-j', 'DNAT', + '--to', fixed_ip) + _confirm_rule("SNATTING", '-t', 'nat', '-s', fixed_ip, '-j', 'SNAT', + '--to', floating_ip) def remove_floating_forward(floating_ip, fixed_ip): """Remove forwarding for floating ip""" - _remove_rule("PREROUTING", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _remove_rule("OUTPUT", "-t nat -d %s -j DNAT --to %s" - % (floating_ip, fixed_ip)) - _remove_rule("SNATTING", "-t nat -s %s -j SNAT --to %s" - % (fixed_ip, floating_ip)) + _remove_rule("PREROUTING", '-t', 'nat', '-d', floating_ip, '-j', 'DNAT', + '--to', fixed_ip) + _remove_rule("OUTPUT", '-t', 'nat', '-d', floating_ip, '-j', 'DNAT', + '--to', fixed_ip) + _remove_rule("SNATTING", '-t', 'nat', '-s', fixed_ip, '-j', 'SNAT', + '--to', floating_ip) def ensure_vlan_bridge(vlan_num, bridge, net_attrs=None): @@ -185,9 +189,9 @@ def ensure_vlan(vlan_num): interface = "vlan%s" % vlan_num if not _device_exists(interface): LOG.debug(_("Starting VLAN inteface %s"), interface) - _execute("sudo vconfig set_name_type VLAN_PLUS_VID_NO_PAD") - _execute("sudo vconfig add %s %s" % (FLAGS.vlan_interface, vlan_num)) - _execute("sudo ip link set %s up" % interface) + _execute('sudo', 'vconfig', 'set_name_type', 'VLAN_PLUS_VID_NO_PAD') + _execute('sudo', 'vconfig', 'add', FLAGS.vlan_interface, vlan_num) + _execute('sudo', 'ip', 'link', 'set', interface, 'up') return interface @@ -206,52 +210,54 @@ def ensure_bridge(bridge, interface, net_attrs=None): """ if not _device_exists(bridge): LOG.debug(_("Starting Bridge interface for %s"), interface) - _execute("sudo brctl addbr %s" % bridge) - _execute("sudo brctl setfd %s 0" % bridge) + _execute('sudo', 'brctl', 'addbr', bridge) + _execute('sudo', 'brctl', 'setfd', bridge, 0) # _execute("sudo brctl setageing %s 10" % bridge) - _execute("sudo brctl stp %s off" % bridge) - _execute("sudo ip link set %s up" % bridge) + _execute('sudo', 'brctl', 'stp', bridge', 'off') + _execute('sudo', 'ip', 'link', 'set', bridge, up) if net_attrs: # NOTE(vish): The ip for dnsmasq has to be the first address on the # bridge for it to respond to reqests properly suffix = net_attrs['cidr'].rpartition('/')[2] - out, err = _execute("sudo ip addr add %s/%s brd %s dev %s" % - (net_attrs['gateway'], - suffix, - net_attrs['broadcast'], - bridge), + out, err = _execute('sudo', 'ip', 'addr', 'add', + "%s/%s" % + (net_attrs['gateway'], suffix), + 'brd', + net-attrs['broadcast'], + 'dev', + bridge, check_exit_code=False) if err and err != "RTNETLINK answers: File exists\n": raise exception.Error("Failed to add ip: %s" % err) if(FLAGS.use_ipv6): - _execute("sudo ip -f inet6 addr change %s dev %s" % - (net_attrs['cidr_v6'], bridge)) + _execute('sudo', 'ip', '-f', 'inet6', 'addr', + 'change', net_attrs['cidr_v6'], + 'dev', bridge) # NOTE(vish): If the public interface is the same as the # bridge, then the bridge has to be in promiscuous # to forward packets properly. if(FLAGS.public_interface == bridge): - _execute("sudo ip link set dev %s promisc on" % bridge) + _execute('sudo', 'ip', 'link', 'set', 'dev', bridge, 'promisc', 'on') if interface: # NOTE(vish): This will break if there is already an ip on the # interface, so we move any ips to the bridge gateway = None - out, err = _execute("sudo route -n") + out, err = _execute('sudo', 'route', '-n') for line in out.split("\n"): fields = line.split() if fields and fields[0] == "0.0.0.0" and fields[-1] == interface: gateway = fields[1] - out, err = _execute("sudo ip addr show dev %s scope global" % - interface) + out, err = _execute('sudo', 'ip', 'addr', 'show', 'dev', interface, + 'scope', 'global') for line in out.split("\n"): fields = line.split() if fields and fields[0] == "inet": params = ' '.join(fields[1:-1]) - _execute("sudo ip addr del %s dev %s" % (params, fields[-1])) - _execute("sudo ip addr add %s dev %s" % (params, bridge)) + _execute('sudo', 'ip', 'addr', 'del', params, 'dev', fields[-1]) + _execute('sudo', 'ip', 'addr', 'add', params, 'dev', bridge) if gateway: - _execute("sudo route add 0.0.0.0 gw %s" % gateway) - out, err = _execute("sudo brctl addif %s %s" % - (bridge, interface), + _execute('sudo', 'route', 'add', '0.0.0.0', 'gw', gateway) + out, err = _execute('sudo', 'brctl', 'addif, bridge, interface, check_exit_code=False) if (err and err != "device %s is already a member of a bridge; can't " @@ -259,18 +265,18 @@ def ensure_bridge(bridge, interface, net_attrs=None): raise exception.Error("Failed to add interface: %s" % err) if FLAGS.use_nova_chains: - (out, err) = _execute("sudo iptables -N nova_forward", + (out, err) = _execute('sudo', 'iptables', '-N', 'nova_forward, check_exit_code=False) if err != 'iptables: Chain already exists.\n': # NOTE(vish): chain didn't exist link chain - _execute("sudo iptables -D FORWARD -j nova_forward", + _execute('sudo', 'iptables, '-D', 'FORWARD', '-j', 'nova_forward', check_exit_code=False) - _execute("sudo iptables -A FORWARD -j nova_forward") + _execute('sudo', 'iptables', '-A', 'FORWARD', '-j', 'nova_forward') - _confirm_rule("FORWARD", "--in-interface %s -j ACCEPT" % bridge) - _confirm_rule("FORWARD", "--out-interface %s -j ACCEPT" % bridge) - _execute("sudo iptables -N nova-local", check_exit_code=False) - _confirm_rule("FORWARD", "-j nova-local") + _confirm_rule("FORWARD", '--in-interface', bridge, '-j', 'ACCEPT') + _confirm_rule("FORWARD", '--out-interface', bridge, '-j', 'ACCEPT') + _execute('sudo', 'iptables', '-N', 'nova-local', check_exit_code=False) + _confirm_rule("FORWARD", '-j', 'nova-local') def get_dhcp_hosts(context, network_id): @@ -304,11 +310,11 @@ def update_dhcp(context, network_id): # if dnsmasq is already running, then tell it to reload if pid: - out, _err = _execute('cat /proc/%d/cmdline' % pid, + out, _err = _execute('cat', "/proc/%d/cmdline" % pid, check_exit_code=False) if conffile in out: try: - _execute('sudo kill -HUP %d' % pid) + _execute('sudo', 'kill', '-HUP', pid) return except Exception as exc: # pylint: disable-msg=W0703 LOG.debug(_("Hupping dnsmasq threw %s"), exc) @@ -349,11 +355,11 @@ interface %s # if radvd is already running, then tell it to reload if pid: - out, _err = _execute('cat /proc/%d/cmdline' + out, _err = _execute('cat', "/proc/%d/cmdline' % pid, check_exit_code=False) if conffile in out: try: - _execute('sudo kill %d' % pid) + _execute('sudo', 'kill', pid) except Exception as exc: # pylint: disable-msg=W0703 LOG.debug(_("killing radvd threw %s"), exc) else: @@ -374,23 +380,23 @@ def _host_dhcp(fixed_ip_ref): fixed_ip_ref['address']) -def _execute(cmd, *args, **kwargs): +def _execute(*cmd, **kwargs): """Wrapper around utils._execute for fake_network""" if FLAGS.fake_network: - LOG.debug("FAKE NET: %s", cmd) + LOG.debug("FAKE NET: %s", ' '.join(cmd)) return "fake", 0 else: - return utils.execute(cmd, *args, **kwargs) + return utils.execute(*cmd, **kwargs) def _device_exists(device): """Check if ethernet device exists""" - (_out, err) = _execute("ip link show dev %s" % device, + (_out, err) = _execute('ip', 'link', 'show', 'dev', device, check_exit_code=False) return not err -def _confirm_rule(chain, cmd, append=False): +def _confirm_rule(chain, *cmd, append=False): """Delete and re-add iptables rule""" if FLAGS.use_nova_chains: chain = "nova_%s" % chain.lower() @@ -398,16 +404,16 @@ def _confirm_rule(chain, cmd, append=False): loc = "-A" else: loc = "-I" - _execute("sudo iptables --delete %s %s" % (chain, cmd), + _execute('sudo', 'iptables', '--delete', chain, *cmd, check_exit_code=False) - _execute("sudo iptables %s %s %s" % (loc, chain, cmd)) + _execute('sudo', 'iptables', loc, chain, *cmd) -def _remove_rule(chain, cmd): +def _remove_rule(chain, *cmd): """Remove iptables rule""" if FLAGS.use_nova_chains: chain = "%s" % chain.lower() - _execute("sudo iptables --delete %s %s" % (chain, cmd)) + _execute('sudo', 'iptables', '--delete', chain, *cmd) def _dnsmasq_cmd(net): @@ -444,7 +450,7 @@ def _stop_dnsmasq(network): if pid: try: - _execute('sudo kill -TERM %d' % pid) + _execute('sudo', 'kill', '-TERM', pid) except Exception as exc: # pylint: disable-msg=W0703 LOG.debug(_("Killing dnsmasq threw %s"), exc) diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index ce1c77210..6d2d8b771 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -343,13 +343,13 @@ def lease_ip(private_ip): private_ip) instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) - cmd = "%s add %s %s fake" % (binpath('nova-dhcpbridge'), - instance_ref['mac_address'], - private_ip) + cmd = (binpath('nova-dhcpbridge'), 'add' + instance_ref['mac_address'], + private_ip, 'fake') env = {'DNSMASQ_INTERFACE': network_ref['bridge'], 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} - (out, err) = utils.execute(cmd, addl_env=env) + (out, err) = utils.execute(*cmd, addl_env=env) LOG.debug("ISSUE_IP: %s, %s ", out, err) @@ -359,11 +359,11 @@ def release_ip(private_ip): private_ip) instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) - cmd = "%s del %s %s fake" % (binpath('nova-dhcpbridge'), - instance_ref['mac_address'], - private_ip) + cmd = (binpath('nova-dhcpbridge'), 'del', + instance_ref['mac_address'], + private_ip, 'fake') env = {'DNSMASQ_INTERFACE': network_ref['bridge'], 'TESTING': '1', 'FLAGFILE': FLAGS.dhcpbridge_flagfile} - (out, err) = utils.execute(cmd, addl_env=env) + (out, err) = utils.execute(*cmd, addl_env=env) LOG.debug("RELEASE_IP: %s, %s ", out, err) diff --git a/nova/utils.py b/nova/utils.py index 40a8d8d8c..c96b85294 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -125,15 +125,15 @@ def fetchfile(url, target): # c.perform() # c.close() # fp.close() - execute("curl","--fail",url,"-o",target) + execute("curl", "--fail", url, "-o", target) -def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (subprocess): %s"), cmd) +def execute(*cmd, process_input=None, addl_env=None, check_exit_code=True): + LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(cmd, stdin=subprocess.PIPE, + obj = subprocess.Popen(*cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) result = None if process_input != None: @@ -148,7 +148,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, stderr=stderr, - cmd=cmd) + cmd=' '.join(cmd)) # NOTE(termie): this appears to be necessary to let the subprocess call # clean something up in between calls, without it two # execute calls in a row hangs the second one @@ -158,7 +158,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): def ssh_execute(ssh, cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (SSH): %s"), cmd) + LOG.debug(_("Running cmd (SSH): %s"), ' '.join(cmd)) if addl_env: raise exception.Error("Environment not supported over SSH") @@ -187,7 +187,7 @@ def ssh_execute(ssh, cmd, process_input=None, raise exception.ProcessExecutionError(exit_code=exit_status, stdout=stdout, stderr=stderr, - cmd=cmd) + cmd=' '.join(cmd)) return (stdout, stderr) @@ -254,7 +254,7 @@ def last_octet(address): def get_my_linklocal(interface): try: - if_str = execute("ip","-f","inet6","-o","addr","show", interface) + if_str = execute("ip", "-f", "inet6", "-o", "addr", "show", interface) condition = "\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link" links = [re.search(condition, x) for x in if_str[0].split('\n')] address = [w.group(1) for w in links if w is not None] diff --git a/nova/virt/disk.py b/nova/virt/disk.py index 2bded07a4..203517275 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -49,10 +49,10 @@ def extend(image, size): file_size = os.path.getsize(image) if file_size >= size: return - utils.execute('truncate -s %s %s' % (size, image)) + utils.execute('truncate', '-s', size, image) # NOTE(vish): attempts to resize filesystem - utils.execute('e2fsck -fp %s' % image, check_exit_code=False) - utils.execute('resize2fs %s' % image, check_exit_code=False) + utils.execute('e2fsck', '-fp', mage, check_exit_code=False) + utils.execute('resize2fs', image, check_exit_code=False) def inject_data(image, key=None, net=None, partition=None, nbd=False): @@ -68,7 +68,7 @@ def inject_data(image, key=None, net=None, partition=None, nbd=False): try: if not partition is None: # create partition - out, err = utils.execute('sudo kpartx -a %s' % device) + out, err = utils.execute('sudo', 'kpartx', '-a', device) if err: raise exception.Error(_('Failed to load partition: %s') % err) mapped_device = '/dev/mapper/%sp%s' % (device.split('/')[-1], @@ -84,13 +84,14 @@ def inject_data(image, key=None, net=None, partition=None, nbd=False): mapped_device) # Configure ext2fs so that it doesn't auto-check every N boots - out, err = utils.execute('sudo tune2fs -c 0 -i 0 %s' % mapped_device) + out, err = utils.execute('sudo', 'tune2fs', + '-c', 0, '-i', 0, mapped_device) tmpdir = tempfile.mkdtemp() try: # mount loopback to dir out, err = utils.execute( - 'sudo mount %s %s' % (mapped_device, tmpdir)) + 'sudo', 'mount', mapped_device, tmpdir) if err: raise exception.Error(_('Failed to mount filesystem: %s') % err) @@ -103,13 +104,13 @@ def inject_data(image, key=None, net=None, partition=None, nbd=False): _inject_net_into_fs(net, tmpdir) finally: # unmount device - utils.execute('sudo umount %s' % mapped_device) + utils.execute('sudo', 'umount', mapped_device) finally: # remove temporary directory - utils.execute('rmdir %s' % tmpdir) + utils.execute('rmdir', tmpdir) if not partition is None: # remove partitions - utils.execute('sudo kpartx -d %s' % device) + utils.execute('sudo', 'kpartx', '-d', device) finally: _unlink_device(device, nbd) @@ -118,7 +119,7 @@ def _link_device(image, nbd): """Link image to device using loopback or nbd""" if nbd: device = _allocate_device() - utils.execute('sudo qemu-nbd -c %s %s' % (device, image)) + utils.execute('sudo', 'qemu-nbd', '-c', device, image) # NOTE(vish): this forks into another process, so give it a chance # to set up before continuuing for i in xrange(FLAGS.timeout_nbd): @@ -127,7 +128,7 @@ def _link_device(image, nbd): time.sleep(1) raise exception.Error(_('nbd device %s did not show up') % device) else: - out, err = utils.execute('sudo losetup --find --show %s' % image) + out, err = utils.execute('sudo', 'losetup', '--find', '--show', image) if err: raise exception.Error(_('Could not attach image to loopback: %s') % err) @@ -137,10 +138,10 @@ def _link_device(image, nbd): def _unlink_device(device, nbd): """Unlink image from device using loopback or nbd""" if nbd: - utils.execute('sudo qemu-nbd -d %s' % device) + utils.execute('sudo', 'qemu-nbd', '-d', device) _free_device(device) else: - utils.execute('sudo losetup --detach %s' % device) + utils.execute('sudo', 'losetup', '--detach', device) _DEVICES = ['/dev/nbd%s' % i for i in xrange(FLAGS.max_nbd_devices)] @@ -170,11 +171,12 @@ def _inject_key_into_fs(key, fs): fs is the path to the base of the filesystem into which to inject the key. """ sshdir = os.path.join(fs, 'root', '.ssh') - utils.execute('sudo mkdir -p %s' % sshdir) # existing dir doesn't matter - utils.execute('sudo chown root %s' % sshdir) - utils.execute('sudo chmod 700 %s' % sshdir) + utils.execute('sudo', 'mkdir', '-p', sshdir) # existing dir doesn't matter + utils.execute('sudo', 'chown', 'root', sshdir) + utils.execute('sudo', 'chmod', '700', sshdir) keyfile = os.path.join(sshdir, 'authorized_keys') - utils.execute('sudo tee -a %s' % keyfile, '\n' + key.strip() + '\n') + # TODO:EWINDISCH: not sure about the following /w execv patch + utils.execute('sudo', 'tee', '-a', keyfile, '\n' + key.strip() + '\n') def _inject_net_into_fs(net, fs): @@ -183,8 +185,8 @@ def _inject_net_into_fs(net, fs): net is the contents of /etc/network/interfaces. """ netdir = os.path.join(os.path.join(fs, 'etc'), 'network') - utils.execute('sudo mkdir -p %s' % netdir) # existing dir doesn't matter - utils.execute('sudo chown root:root %s' % netdir) - utils.execute('sudo chmod 755 %s' % netdir) + utils.execute('sudo', 'mkdir', '-p', netdir) # existing dir doesn't matter + utils.execute('sudo', 'chown', 'root:root', netdir) + utils.execute('sudo', 'chmod', 755, netdir) netfile = os.path.join(netdir, 'interfaces') - utils.execute('sudo tee %s' % netfile, net) + utils.execute('sudo', 'tee', netfile, net) diff --git a/nova/virt/images.py b/nova/virt/images.py index 7a6fef330..4b11d1667 100644 --- a/nova/virt/images.py +++ b/nova/virt/images.py @@ -94,8 +94,7 @@ def _fetch_s3_image(image, path, user, project): cmd += ['-H', '\'%s: %s\'' % (k, v)] cmd += ['-o', path] - cmd_out = ' '.join(cmd) - return utils.execute(cmd_out) + return utils.execute(*cmd) def _fetch_local_image(image, path, user, project): @@ -103,7 +102,7 @@ def _fetch_local_image(image, path, user, project): if sys.platform.startswith('win'): return shutil.copy(source, path) else: - return utils.execute('cp %s %s' % (source, path)) + return utils.execute('cp', source, path) def _image_path(path): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 4e0fd106f..464ec475c 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -438,8 +438,10 @@ class LibvirtConnection(object): if virsh_output.startswith('/dev/'): LOG.info(_("cool, it's a device")) - out, err = utils.execute("sudo dd if=%s iflag=nonblock" % - virsh_output, check_exit_code=False) + out, err = utils.execute('sudo', 'dd', + "if=%s" % virsh_output, + 'iflag=nonblock', + check_exit_code=False) return out else: return '' @@ -461,11 +463,11 @@ class LibvirtConnection(object): console_log = os.path.join(FLAGS.instances_path, instance['name'], 'console.log') - utils.execute('sudo chown %d %s' % (os.getuid(), console_log)) + utils.execute('sudo', 'chown', s.getuid(), console_log) if FLAGS.libvirt_type == 'xen': # Xen is special - virsh_output = utils.execute("virsh ttyconsole %s" % + virsh_output = utils.execute('virsh', 'ttyconsole', instance['name']) data = self._flush_xen_console(virsh_output) fpath = self._append_to_file(data, console_log) @@ -482,7 +484,10 @@ class LibvirtConnection(object): port = random.randint(int(start_port), int(end_port)) # netcat will exit with 0 only if the port is in use, # so a nonzero return value implies it is unused - cmd = 'netcat 0.0.0.0 %s -w 1 Date: Tue, 8 Mar 2011 01:08:13 -0500 Subject: Fix todo comment --- nova/virt/libvirt_conn.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index e1cd75306..a5256bbc2 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -486,8 +486,9 @@ class LibvirtConnection(object): # netcat will exit with 0 only if the port is in use, # so a nonzero return value implies it is unused - # TODO:ewindisch: subprocess lets us do this... - # but utils.execute abstracts it away from us + # TODO(ewindisch): broken /w execvp patch. + # subprocess lets us do this, but utils.execute + # abstracts it away from us cmd = 'netcat', '0.0.0.0', port, '-w', '1', ' Date: Wed, 9 Mar 2011 00:30:05 -0500 Subject: execvp: almost passes tests --- nova/api/ec2/cloud.py | 2 +- nova/network/linux_net.py | 21 +++++++++++---------- nova/tests/test_network.py | 2 +- nova/tests/test_virt.py | 11 ++++++----- nova/utils.py | 19 +++++++++++++------ nova/virt/libvirt_conn.py | 11 ++++++----- nova/virt/xenapi/vm_utils.py | 6 ++---- nova/volume/driver.py | 5 +++-- 8 files changed, 43 insertions(+), 34 deletions(-) diff --git a/nova/api/ec2/cloud.py b/nova/api/ec2/cloud.py index 0d22a3f46..b7d72d1c1 100644 --- a/nova/api/ec2/cloud.py +++ b/nova/api/ec2/cloud.py @@ -115,7 +115,7 @@ class CloudController(object): start = os.getcwd() os.chdir(FLAGS.ca_path) # TODO(vish): Do this with M2Crypto instead - utils.runthis(_("Generating root CA: %s"), "sh genrootca.sh") + utils.runthis(_("Generating root CA: %s"), "sh", "genrootca.sh") os.chdir(start) def _get_mpi_data(self, context, project_id): diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index ad019a8c0..b66a1adb7 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -139,14 +139,14 @@ def init_host(): def bind_floating_ip(floating_ip, check_exit_code=True): """Bind ip to public interface""" _execute('sudo', 'ip', 'addr', 'add', floating_ip, - 'dev', FLAGS.public_interface), + 'dev', FLAGS.public_interface, check_exit_code=check_exit_code) def unbind_floating_ip(floating_ip): """Unbind a public ip from public interface""" _execute('sudo', 'ip', 'addr', 'del', floating_ip, - 'dev', FLAGS.public_interface)) + 'dev', FLAGS.public_interface) def ensure_vlan_forward(public_ip, port, private_ip): @@ -213,7 +213,7 @@ def ensure_bridge(bridge, interface, net_attrs=None): _execute('sudo', 'brctl', 'addbr', bridge) _execute('sudo', 'brctl', 'setfd', bridge, 0) # _execute("sudo brctl setageing %s 10" % bridge) - _execute('sudo', 'brctl', 'stp', bridge', 'off') + _execute('sudo', 'brctl', 'stp', bridge, 'off') _execute('sudo', 'ip', 'link', 'set', bridge, up) if net_attrs: # NOTE(vish): The ip for dnsmasq has to be the first address on the @@ -223,7 +223,7 @@ def ensure_bridge(bridge, interface, net_attrs=None): "%s/%s" % (net_attrs['gateway'], suffix), 'brd', - net-attrs['broadcast'], + net_attrs['broadcast'], 'dev', bridge, check_exit_code=False) @@ -257,7 +257,7 @@ def ensure_bridge(bridge, interface, net_attrs=None): _execute('sudo', 'ip', 'addr', 'add', params, 'dev', bridge) if gateway: _execute('sudo', 'route', 'add', '0.0.0.0', 'gw', gateway) - out, err = _execute('sudo', 'brctl', 'addif, bridge, interface, + out, err = _execute('sudo', 'brctl', 'addif', bridge, interface, check_exit_code=False) if (err and err != "device %s is already a member of a bridge; can't " @@ -265,11 +265,11 @@ def ensure_bridge(bridge, interface, net_attrs=None): raise exception.Error("Failed to add interface: %s" % err) if FLAGS.use_nova_chains: - (out, err) = _execute('sudo', 'iptables', '-N', 'nova_forward, + (out, err) = _execute('sudo', 'iptables', '-N', 'nova_forward', check_exit_code=False) if err != 'iptables: Chain already exists.\n': # NOTE(vish): chain didn't exist link chain - _execute('sudo', 'iptables, '-D', 'FORWARD', '-j', 'nova_forward', + _execute('sudo', 'iptables', '-D', 'FORWARD', '-j', 'nova_forward', check_exit_code=False) _execute('sudo', 'iptables', '-A', 'FORWARD', '-j', 'nova_forward') @@ -355,7 +355,7 @@ interface %s # if radvd is already running, then tell it to reload if pid: - out, _err = _execute('cat', "/proc/%d/cmdline' + out, _err = _execute('cat', '/proc/%d/cmdline' % pid, check_exit_code=False) if conffile in out: try: @@ -383,7 +383,7 @@ def _host_dhcp(fixed_ip_ref): def _execute(*cmd, **kwargs): """Wrapper around utils._execute for fake_network""" if FLAGS.fake_network: - LOG.debug("FAKE NET: %s", ' '.join(cmd)) + LOG.debug("FAKE NET: %s", " ".join(map(str, cmd))) return "fake", 0 else: return utils.execute(*cmd, **kwargs) @@ -396,7 +396,8 @@ def _device_exists(device): return not err -def _confirm_rule(chain, *cmd, append=False): +def _confirm_rule(chain, *cmd, **kwargs): + append=kwargs.get('append',False) """Delete and re-add iptables rule""" if FLAGS.use_nova_chains: chain = "nova_%s" % chain.lower() diff --git a/nova/tests/test_network.py b/nova/tests/test_network.py index 6d2d8b771..19099ff4c 100644 --- a/nova/tests/test_network.py +++ b/nova/tests/test_network.py @@ -343,7 +343,7 @@ def lease_ip(private_ip): private_ip) instance_ref = db.fixed_ip_get_instance(context.get_admin_context(), private_ip) - cmd = (binpath('nova-dhcpbridge'), 'add' + cmd = (binpath('nova-dhcpbridge'), 'add', instance_ref['mac_address'], private_ip, 'fake') env = {'DNSMASQ_INTERFACE': network_ref['bridge'], diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index f151ae911..7f1ad002e 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -315,15 +315,16 @@ class IptablesFirewallTestCase(test.TestCase): instance_ref = db.instance_get(admin_ctxt, instance_ref['id']) # self.fw.add_instance(instance_ref) - def fake_iptables_execute(cmd, process_input=None): - if cmd == 'sudo ip6tables-save -t filter': + def fake_iptables_execute(*cmd, **kwargs): + process_input=kwargs.get('process_input', None) + if cmd == ('sudo', 'ip6tables-save', '-t', 'filter'): return '\n'.join(self.in6_rules), None - if cmd == 'sudo iptables-save -t filter': + if cmd == ('sudo', 'iptables-save', '-t', 'filter'): return '\n'.join(self.in_rules), None - if cmd == 'sudo iptables-restore': + if cmd == ('sudo', 'iptables-restore'): self.out_rules = process_input.split('\n') return '', '' - if cmd == 'sudo ip6tables-restore': + if cmd == ('sudo', 'ip6tables-restore'): self.out6_rules = process_input.split('\n') return '', '' self.fw.execute = fake_iptables_execute diff --git a/nova/utils.py b/nova/utils.py index c96b85294..9b51f8b40 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -128,13 +128,20 @@ def fetchfile(url, target): execute("curl", "--fail", url, "-o", target) -def execute(*cmd, process_input=None, addl_env=None, check_exit_code=True): +def execute(*cmd, **kwargs): + process_input=kwargs.get('process_input', None) + addl_env=kwargs.get('addl_env', None) + check_exit_code=kwargs.get('check_exit_code', True) + stdin=kwargs.get('stdin', subprocess.PIPE) + stdout=kwargs.get('stdout', subprocess.PIPE) + stderr=kwargs.get('stderr', subprocess.PIPE) + LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(*cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + obj = subprocess.Popen(cmd, stdin=stdin, + stdout=stdout, stderr=stderr, env=env) result = None if process_input != None: result = obj.communicate(process_input) @@ -220,9 +227,9 @@ def debug(arg): return arg -def runthis(prompt, cmd, check_exit_code=True): - LOG.debug(_("Running %s"), (cmd)) - rv, err = execute(cmd, check_exit_code=check_exit_code) +def runthis(prompt, *cmd, **kwargs): + LOG.debug(_("Running %s"), (" ".join(cmd))) + rv, err = execute(*cmd, **kwargs) def generate_uid(topic, size=8): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index a5256bbc2..76f31f91a 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -540,8 +540,8 @@ class LibvirtConnection(object): if not os.path.exists(base): fn(target=base, *args, **kwargs) if cow: - utils.execute('qemu-img', 'create', '-f', 'qcow2', "'-o'', - "cluster_size=2M,backing_file=%s" % base, + utils.execute('qemu-img', 'create', '-f', 'qcow2', '-o', + 'cluster_size=2M,backing_file=%s' % base, target) else: utils.execute('cp', base, target) @@ -554,7 +554,7 @@ class LibvirtConnection(object): def _create_local(self, target, local_gb): """Create a blank image of specified size""" - utils.execute('truncate', target, '-s', "%dG" local_gb) + utils.execute('truncate', target, '-s', "%dG" % local_gb) # TODO(vish): should we format disk by default? def _create_image(self, inst, libvirt_xml, suffix='', disk_images=None): @@ -565,7 +565,7 @@ class LibvirtConnection(object): fname + suffix) # ensure directories exist and are writable - utils.execute('mkdir', '-p', basepath(suffix='') + utils.execute('mkdir', '-p', basepath(suffix='')) LOG.info(_('instance %s: Creating image'), inst['name']) f = open(basepath('libvirt.xml'), 'w') @@ -1245,7 +1245,8 @@ class IptablesFirewallDriver(FirewallDriver): self.apply_ruleset() def apply_ruleset(self): - current_filter, _ = self.execute('sudo iptables-save -t filter') + current_filter, _ = self.execute('sudo', 'iptables-save', + '-t', 'filter') current_lines = current_filter.split('\n') new_filter = self.modify_rules(current_lines, 4) self.execute('sudo', 'iptables-restore', diff --git a/nova/virt/xenapi/vm_utils.py b/nova/virt/xenapi/vm_utils.py index 8fdb658fb..6ba13f980 100644 --- a/nova/virt/xenapi/vm_utils.py +++ b/nova/virt/xenapi/vm_utils.py @@ -915,10 +915,8 @@ def _write_partition(virtual_size, dev): LOG.debug(_('Writing partition table %(primary_first)d %(primary_last)d' ' to %(dest)s...') % locals()) - def execute(*cmd, process_input=None, check_exit_code=True): - return utils.execute(*cmd, - process_input=process_input, - check_exit_code=check_exit_code) + def execute(*cmd, **kwargs): + return utils.execute(*cmd, **kwargs) execute('parted', '--script', dest, 'mklabel', 'msdos') execute('parted', '--script', dest, 'mkpart', 'primary', diff --git a/nova/volume/driver.py b/nova/volume/driver.py index 220c9ef9d..e9bdf162f 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -104,7 +104,8 @@ class VolumeDriver(object): def delete_volume(self, volume): """Deletes a logical volume.""" try: - self._try_execute('sudo', 'lvdisplay', '%s/%s" % + self._try_execute('sudo', 'lvdisplay', + '%s/%s' % (FLAGS.volume_group, volume['name'])) except Exception as e: @@ -550,7 +551,7 @@ class SheepdogDriver(VolumeDriver): else: sizestr = '%sG' % volume['size'] self._try_execute('qemu-img', 'create', - "sheepdog:%s" %s" % volume['name'], + "sheepdog:%s" % volume['name'], sizestr) def delete_volume(self, volume): -- cgit From 1d7358e70379607c9cce02307f4336efbd135a5d Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 01:26:53 -0500 Subject: execvp: unit tests pass --- nova/crypto.py | 2 +- nova/utils.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/nova/crypto.py b/nova/crypto.py index dd24723b8..20bb570a5 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -196,7 +196,7 @@ def generate_x509_cert(user_id, project_id, bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.abspath(os.path.join(tmpdir, 'temp.key')) csrfile = os.path.join(tmpdir, 'temp.csr') - utils.execute('openssl', 'genrsa', '-out', keyfile, bits) + utils.execute('openssl', 'genrsa', '-out', keyfile, str(bits)) utils.execute('openssl', 'req', '-new', '-key', keyfile, '-out', csrfile, '-batch', '-subj', subject) private_key = open(keyfile).read() diff --git a/nova/utils.py b/nova/utils.py index 9b51f8b40..7ddf056ea 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -135,6 +135,7 @@ def execute(*cmd, **kwargs): stdin=kwargs.get('stdin', subprocess.PIPE) stdout=kwargs.get('stdout', subprocess.PIPE) stderr=kwargs.get('stderr', subprocess.PIPE) + cmd=map(str,cmd) LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() -- cgit From 23369a63f4b74fb64bf57554a3fd8b15e3e2b49c Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 14:31:23 -0500 Subject: Fixes uses of process_input --- nova/utils.py | 4 ++-- nova/virt/disk.py | 4 ++-- nova/virt/libvirt_conn.py | 11 ++++------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/nova/utils.py b/nova/utils.py index 7ddf056ea..0937522ec 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -131,7 +131,7 @@ def fetchfile(url, target): def execute(*cmd, **kwargs): process_input=kwargs.get('process_input', None) addl_env=kwargs.get('addl_env', None) - check_exit_code=kwargs.get('check_exit_code', True) + check_exit_code=kwargs.get('check_exit_code', 0) stdin=kwargs.get('stdin', subprocess.PIPE) stdout=kwargs.get('stdout', subprocess.PIPE) stderr=kwargs.get('stderr', subprocess.PIPE) @@ -151,7 +151,7 @@ def execute(*cmd, **kwargs): obj.stdin.close() if obj.returncode: LOG.debug(_("Result was %s") % obj.returncode) - if check_exit_code and obj.returncode != 0: + if check_exit_code is not None and obj.returncode != check_exit_code: (stdout, stderr) = result raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, diff --git a/nova/virt/disk.py b/nova/virt/disk.py index 203517275..a54cda003 100644 --- a/nova/virt/disk.py +++ b/nova/virt/disk.py @@ -175,8 +175,8 @@ def _inject_key_into_fs(key, fs): utils.execute('sudo', 'chown', 'root', sshdir) utils.execute('sudo', 'chmod', '700', sshdir) keyfile = os.path.join(sshdir, 'authorized_keys') - # TODO:EWINDISCH: not sure about the following /w execv patch - utils.execute('sudo', 'tee', '-a', keyfile, '\n' + key.strip() + '\n') + utils.execute('sudo', 'tee', '-a', keyfile, + process_input='\n' + key.strip() + '\n') def _inject_net_into_fs(net, fs): diff --git a/nova/virt/libvirt_conn.py b/nova/virt/libvirt_conn.py index 76f31f91a..6b555ecbb 100644 --- a/nova/virt/libvirt_conn.py +++ b/nova/virt/libvirt_conn.py @@ -485,13 +485,10 @@ class LibvirtConnection(object): port = random.randint(int(start_port), int(end_port)) # netcat will exit with 0 only if the port is in use, # so a nonzero return value implies it is unused - - # TODO(ewindisch): broken /w execvp patch. - # subprocess lets us do this, but utils.execute - # abstracts it away from us - cmd = 'netcat', '0.0.0.0', port, '-w', '1', ' Date: Wed, 9 Mar 2011 15:08:11 -0500 Subject: renaming wsgi.Request.best_match to best_match_content_type; correcting calls to that function in code from trunk --- nova/api/direct.py | 2 +- nova/api/openstack/__init__.py | 2 +- nova/api/openstack/faults.py | 2 +- nova/api/openstack/servers.py | 2 +- nova/tests/api/openstack/common.py | 1 + nova/tests/api/test_wsgi.py | 18 +++++++++--------- nova/wsgi.py | 4 ++-- 7 files changed, 16 insertions(+), 15 deletions(-) diff --git a/nova/api/direct.py b/nova/api/direct.py index 1d699f947..dfca250e0 100644 --- a/nova/api/direct.py +++ b/nova/api/direct.py @@ -206,7 +206,7 @@ class ServiceWrapper(wsgi.Controller): params = dict([(str(k), v) for (k, v) in params.iteritems()]) result = method(context, **params) if type(result) is dict or type(result) is list: - return self._serialize(result, req.best_match()) + return self._serialize(result, req.best_match_content_type()) else: return result diff --git a/nova/api/openstack/__init__.py b/nova/api/openstack/__init__.py index 6e1a2a06c..197fcc619 100644 --- a/nova/api/openstack/__init__.py +++ b/nova/api/openstack/__init__.py @@ -125,5 +125,5 @@ class Versions(wsgi.Application): "application/xml": { "attributes": dict(version=["status", "id"])}} - content_type = req.best_match() + content_type = req.best_match_content_type() return wsgi.Serializer(metadata).serialize(response, content_type) diff --git a/nova/api/openstack/faults.py b/nova/api/openstack/faults.py index 075fdb997..2fd733299 100644 --- a/nova/api/openstack/faults.py +++ b/nova/api/openstack/faults.py @@ -58,6 +58,6 @@ class Fault(webob.exc.HTTPException): # 'code' is an attribute on the fault tag itself metadata = {'application/xml': {'attributes': {fault_name: 'code'}}} serializer = wsgi.Serializer(metadata) - content_type = req.best_match() + content_type = req.best_match_content_type() self.wrapped_exc.body = serializer.serialize(fault_data, content_type) return self.wrapped_exc diff --git a/nova/api/openstack/servers.py b/nova/api/openstack/servers.py index 8dd078a31..25c667532 100644 --- a/nova/api/openstack/servers.py +++ b/nova/api/openstack/servers.py @@ -217,7 +217,7 @@ class Controller(wsgi.Controller): 'rebuild': self._action_rebuild, } - input_dict = self._deserialize(req.body, req) + input_dict = self._deserialize(req.body, req.get_content_type()) for key in actions.keys(): if key in input_dict: return actions[key](input_dict, req, id) diff --git a/nova/tests/api/openstack/common.py b/nova/tests/api/openstack/common.py index 3f9c7d3cf..74bb8729a 100644 --- a/nova/tests/api/openstack/common.py +++ b/nova/tests/api/openstack/common.py @@ -28,6 +28,7 @@ def webob_factory(url): def web_request(url, method=None, body=None): req = webob.Request.blank("%s%s" % (base_url, url)) if method: + req.content_type = "application/json" req.method = method if body: req.body = json.dumps(body) diff --git a/nova/tests/api/test_wsgi.py b/nova/tests/api/test_wsgi.py index 7c0135656..b1a849cf9 100644 --- a/nova/tests/api/test_wsgi.py +++ b/nova/tests/api/test_wsgi.py @@ -139,48 +139,48 @@ class RequestTest(test.TestCase): def test_content_type_from_accept_xml(self): request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/xml" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/xml") request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/json" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = "application/xml, application/json" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") request = wsgi.Request.blank('/tests/123') request.headers["Accept"] = \ "application/json; q=0.3, application/xml; q=0.9" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/xml") def test_content_type_from_query_extension(self): request = wsgi.Request.blank('/tests/123.xml') - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/xml") request = wsgi.Request.blank('/tests/123.json') - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") request = wsgi.Request.blank('/tests/123.invalid') - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") def test_content_type_accept_and_query_extension(self): request = wsgi.Request.blank('/tests/123.xml') request.headers["Accept"] = "application/json" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/xml") def test_content_type_accept_default(self): request = wsgi.Request.blank('/tests/123.unsupported') request.headers["Accept"] = "application/unsupported1" - result = request.best_match() + result = request.best_match_content_type() self.assertEqual(result, "application/json") diff --git a/nova/wsgi.py b/nova/wsgi.py index c3e08522d..2d18da8fb 100644 --- a/nova/wsgi.py +++ b/nova/wsgi.py @@ -85,7 +85,7 @@ class Server(object): class Request(webob.Request): - def best_match(self): + def best_match_content_type(self): """ Determine the most acceptable content-type based on the query extension then the Accept header @@ -354,7 +354,7 @@ class Controller(object): result = method(**arg_dict) if type(result) is dict: - content_type = req.best_match() + content_type = req.best_match_content_type() body = self._serialize(result, content_type) response = webob.Response() -- cgit From fc9840bae6200c8f89fb8a3ba0ab45663c872b3c Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 15:33:20 -0500 Subject: execvp passes pep8 --- nova/console/xvp.py | 6 +++--- nova/crypto.py | 3 ++- nova/network/linux_net.py | 19 ++++++++++++------- nova/tests/test_virt.py | 2 +- nova/utils.py | 19 ++++++++++--------- nova/volume/driver.py | 4 ++-- 6 files changed, 30 insertions(+), 23 deletions(-) diff --git a/nova/console/xvp.py b/nova/console/xvp.py index 271dffa54..68d8c8565 100644 --- a/nova/console/xvp.py +++ b/nova/console/xvp.py @@ -134,9 +134,9 @@ class XVPConsoleProxy(object): logging.debug(_("Starting xvp")) try: utils.execute('xvp', - '-p',FLAGS.console_xvp_pid, - '-c',FLAGS.console_xvp_conf, - '-l',FLAGS.console_xvp_log) + '-p', FLAGS.console_xvp_pid, + '-c', FLAGS.console_xvp_conf, + '-l', FLAGS.console_xvp_log) except exception.ProcessExecutionError, err: logging.error(_("Error starting xvp: %s") % err) diff --git a/nova/crypto.py b/nova/crypto.py index 20bb570a5..717ea0041 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -120,7 +120,8 @@ def generate_key_pair(bits=1024): # bio = M2Crypto.BIO.MemoryBuffer() # key.save_pub_key_bio(bio) # public_key = bio.read() - # public_key, err = execute('ssh-keygen', '-y', '-f', '/dev/stdin', private_key) + # public_key, err = execute('ssh-keygen', '-y', '-f', + # '/dev/stdin', private_key) return (private_key, public_key, fingerprint) diff --git a/nova/network/linux_net.py b/nova/network/linux_net.py index b66a1adb7..228a4d9ea 100644 --- a/nova/network/linux_net.py +++ b/nova/network/linux_net.py @@ -68,7 +68,8 @@ def metadata_forward(): _confirm_rule("PREROUTING", '-t', 'nat', '-s', '0.0.0.0/0', '-d', '169.254.169.254/32', '-p', 'tcp', '-m', 'tcp', '--dport', '80', '-j', 'DNAT', - '--to-destination', '%s:%s' % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) + '--to-destination', + '%s:%s' % (FLAGS.ec2_dmz_host, FLAGS.ec2_port)) def init_host(): @@ -86,7 +87,8 @@ def init_host(): _execute('sudo', 'iptables', '-D', 'FORWARD', '-j', 'nova_forward', check_exit_code=False) _execute('sudo', 'iptables', '-A', 'FORWARD', '-j', 'nova_forward') - _execute('sudo', 'iptables', '-N', 'nova_output', check_exit_code=False) + _execute('sudo', 'iptables', '-N', 'nova_output', + check_exit_code=False) _execute('sudo', 'iptables', '-D', 'OUTPUT', '-j', 'nova_output', check_exit_code=False) _execute('sudo', 'iptables', '-A', 'OUTPUT', '-j', 'nova_output') @@ -220,7 +222,7 @@ def ensure_bridge(bridge, interface, net_attrs=None): # bridge for it to respond to reqests properly suffix = net_attrs['cidr'].rpartition('/')[2] out, err = _execute('sudo', 'ip', 'addr', 'add', - "%s/%s" % + "%s/%s" % (net_attrs['gateway'], suffix), 'brd', net_attrs['broadcast'], @@ -237,7 +239,8 @@ def ensure_bridge(bridge, interface, net_attrs=None): # bridge, then the bridge has to be in promiscuous # to forward packets properly. if(FLAGS.public_interface == bridge): - _execute('sudo', 'ip', 'link', 'set', 'dev', bridge, 'promisc', 'on') + _execute('sudo', 'ip', 'link', 'set', + 'dev', bridge, 'promisc', 'on') if interface: # NOTE(vish): This will break if there is already an ip on the # interface, so we move any ips to the bridge @@ -253,8 +256,10 @@ def ensure_bridge(bridge, interface, net_attrs=None): fields = line.split() if fields and fields[0] == "inet": params = ' '.join(fields[1:-1]) - _execute('sudo', 'ip', 'addr', 'del', params, 'dev', fields[-1]) - _execute('sudo', 'ip', 'addr', 'add', params, 'dev', bridge) + _execute('sudo', 'ip', 'addr', + 'del', params, 'dev', fields[-1]) + _execute('sudo', 'ip', 'addr', + 'add', params, 'dev', bridge) if gateway: _execute('sudo', 'route', 'add', '0.0.0.0', 'gw', gateway) out, err = _execute('sudo', 'brctl', 'addif', bridge, interface, @@ -397,7 +402,7 @@ def _device_exists(device): def _confirm_rule(chain, *cmd, **kwargs): - append=kwargs.get('append',False) + append = kwargs.get('append', False) """Delete and re-add iptables rule""" if FLAGS.use_nova_chains: chain = "nova_%s" % chain.lower() diff --git a/nova/tests/test_virt.py b/nova/tests/test_virt.py index 7f1ad002e..dfa607f14 100644 --- a/nova/tests/test_virt.py +++ b/nova/tests/test_virt.py @@ -316,7 +316,7 @@ class IptablesFirewallTestCase(test.TestCase): # self.fw.add_instance(instance_ref) def fake_iptables_execute(*cmd, **kwargs): - process_input=kwargs.get('process_input', None) + process_input = kwargs.get('process_input', None) if cmd == ('sudo', 'ip6tables-save', '-t', 'filter'): return '\n'.join(self.in6_rules), None if cmd == ('sudo', 'iptables-save', '-t', 'filter'): diff --git a/nova/utils.py b/nova/utils.py index 0937522ec..3a4ec3c6a 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -40,7 +40,7 @@ import netaddr from eventlet import event from eventlet import greenthread from eventlet.green import subprocess - +None from nova import exception from nova.exception import ProcessExecutionError from nova import log as logging @@ -129,13 +129,13 @@ def fetchfile(url, target): def execute(*cmd, **kwargs): - process_input=kwargs.get('process_input', None) - addl_env=kwargs.get('addl_env', None) - check_exit_code=kwargs.get('check_exit_code', 0) - stdin=kwargs.get('stdin', subprocess.PIPE) - stdout=kwargs.get('stdout', subprocess.PIPE) - stderr=kwargs.get('stderr', subprocess.PIPE) - cmd=map(str,cmd) + process_input = kwargs.get('process_input', None) + addl_env = kwargs.get('addl_env', None) + check_exit_code = kwargs.get('check_exit_code', 0) + stdin = kwargs.get('stdin', subprocess.PIPE) + stdout = kwargs.get('stdout', subprocess.PIPE) + stderr = kwargs.get('stderr', subprocess.PIPE) + cmd = map(str, cmd) LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() @@ -151,7 +151,8 @@ def execute(*cmd, **kwargs): obj.stdin.close() if obj.returncode: LOG.debug(_("Result was %s") % obj.returncode) - if check_exit_code is not None and obj.returncode != check_exit_code: + if type(check_exit_code) == types.IntType \ + and obj.returncode != check_exit_code: (stdout, stderr) = result raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, diff --git a/nova/volume/driver.py b/nova/volume/driver.py index e9bdf162f..45cc800e7 100644 --- a/nova/volume/driver.py +++ b/nova/volume/driver.py @@ -112,7 +112,7 @@ class VolumeDriver(object): # If the volume isn't present, then don't attempt to delete return True - self._try_execute('sudo', 'lvremove', '-f',"%s/%s" % + self._try_execute('sudo', 'lvremove', '-f', "%s/%s" % (FLAGS.volume_group, volume['name'])) @@ -256,7 +256,7 @@ class ISCSIDriver(VolumeDriver): self._sync_exec('sudo', 'ietadm', '--op', 'new', "--tid=%s" % iscsi_target, '--params', - "Name=%s" % iscsi-name, + "Name=%s" % iscsi_name, check_exit_code=False) self._sync_exec('sudo', 'ietadm', '--op', 'new', "--tid=%s" % iscsi_target, -- cgit From e8554da80ac916f168461cb48078488700081c02 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 16:44:48 -0500 Subject: execvp: cleanup. --- nova/crypto.py | 6 +++--- .../networking/etc/xensource/scripts/vif_rules.py | 22 +++++++++++----------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/nova/crypto.py b/nova/crypto.py index 717ea0041..2a8d4abca 100644 --- a/nova/crypto.py +++ b/nova/crypto.py @@ -105,7 +105,7 @@ def generate_key_pair(bits=1024): tmpdir = tempfile.mkdtemp() keyfile = os.path.join(tmpdir, 'temp') - utils.execute('ssh-keygen', '-q', '-b', '%d' % bits, '-N', '', + utils.execute('ssh-keygen', '-q', '-b', bits, '-N', '', '-f', keyfile) (out, err) = utils.execute('ssh-keygen', '-q', '-l', '-f', '%s.pub' % (keyfile)) @@ -147,9 +147,9 @@ def revoke_cert(project_id, file_name): os.chdir(ca_folder(project_id)) # NOTE(vish): potential race condition here utils.execute('openssl', 'ca', '-config', './openssl.cnf', '-revoke', - '%s' % file_name) + file_name) utils.execute('openssl', 'ca', '-gencrl', '-config', './openssl.cnf', - '-out', '%s' % FLAGS.crl_file) + '-out', FLAGS.crl_file) os.chdir(start) diff --git a/plugins/xenserver/networking/etc/xensource/scripts/vif_rules.py b/plugins/xenserver/networking/etc/xensource/scripts/vif_rules.py index 2c34f7b1d..d2b2d61e6 100755 --- a/plugins/xenserver/networking/etc/xensource/scripts/vif_rules.py +++ b/plugins/xenserver/networking/etc/xensource/scripts/vif_rules.py @@ -52,7 +52,7 @@ def main(dom_id, command, only_this_vif=None): apply_iptables_rules(command, params) -def execute(command, return_stdout=False): +def execute(*command, return_stdout=False): devnull = open(os.devnull, 'w') proc = subprocess.Popen(command, close_fds=True, stdout=subprocess.PIPE, stderr=devnull) @@ -110,26 +110,26 @@ def apply_arptables_rules(command, params): def apply_ebtables_rules(command, params): ebtables = lambda *rule: execute("/sbin/ebtables", *rule) - ebtables('-D', 'FORWARD', '-p', '0806', '-o', '%(VIF)s' % params, - '--arp-ip-dst', '%(IP)s' % params, + ebtables('-D', 'FORWARD', '-p', '0806', '-o', params['VIF'], + '--arp-ip-dst', params['IP'], '-j', 'ACCEPT') ebtables('-D', 'FORWARD', '-p', '0800', '-o', - '%(VIF)s' % params, '--ip-dst', '%(IP)s' % params, + params['VIF'], '--ip-dst', params['IP'], '-j', 'ACCEPT') if command == 'online': ebtables('-A', 'FORWARD', '-p', '0806', - '-o', '%(VIF)s' % params - '--arp-ip-dst', '%(IP)s' % params, + '-o', params['VIF'], + '--arp-ip-dst', params['IP'], '-j', 'ACCEPT') ebtables('-A', 'FORWARD', '-p', '0800', - '-o', '%(VIF)s' % params, - '--ip-dst', '%(IP)s' % params, + '-o', params['VIF'], + '--ip-dst', params['IP'], '-j', 'ACCEPT') - ebtables('-D', 'FORWARD', '-s', '!', '%(MAC)s' % params, - '-i', '%(VIF)s' % params, '-j', 'DROP') + ebtables('-D', 'FORWARD', '-s', '!', params['MAC'], + '-i', params['VIF'], '-j', 'DROP') if command == 'online': - ebtables('-I', 'FORWARD', '1', '-s', '!', '%(MAC)s' % params, + ebtables('-I', 'FORWARD', '1', '-s', '!', params['MAC'], '-i', '%(VIF)s', '-j', 'DROP') -- cgit