From 5812a95736b9a16733b99700e8664dd29ae34def Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Fri, 18 Feb 2011 22:10:06 +0100 Subject: Introduce IptablesManager in linux_net. Port every use of iptables in linux_net to it. --- nova/utils.py | 61 ++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 35 insertions(+), 26 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index ba71ebf39..bf3a4b098 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -124,32 +124,41 @@ def fetchfile(url, target): execute("curl --fail %s -o %s" % (url, target)) -def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (subprocess): %s"), cmd) - env = os.environ.copy() - if addl_env: - env.update(addl_env) - obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) - result = None - if process_input != None: - result = obj.communicate(process_input) - else: - result = obj.communicate() - obj.stdin.close() - if obj.returncode: - LOG.debug(_("Result was %s") % obj.returncode) - if check_exit_code and obj.returncode != 0: - (stdout, stderr) = result - raise ProcessExecutionError(exit_code=obj.returncode, - stdout=stdout, - stderr=stderr, - cmd=cmd) - # NOTE(termie): this appears to be necessary to let the subprocess call - # clean something up in between calls, without it two - # execute calls in a row hangs the second one - greenthread.sleep(0) - return result +def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, attempts=1): + while attempts > 0: + attempts -= 1 + try: + LOG.debug(_("Running cmd (subprocess): %s"), cmd) + env = os.environ.copy() + if addl_env: + env.update(addl_env) + obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + result = None + if process_input != None: + result = obj.communicate(process_input) + else: + result = obj.communicate() + obj.stdin.close() + if obj.returncode: + LOG.debug(_("Result was %s") % obj.returncode) + if check_exit_code and obj.returncode != 0: + (stdout, stderr) = result + raise ProcessExecutionError(exit_code=obj.returncode, + stdout=stdout, + stderr=stderr, + cmd=cmd) + # NOTE(termie): this appears to be necessary to let the subprocess call + # clean something up in between calls, without it two + # execute calls in a row hangs the second one + greenthread.sleep(0) + return result + except ProcessExecutionError: + if not attempts: + raise + else: + greenthread.sleep(random.randint(50,300)/100) + pass def ssh_execute(ssh, cmd, process_input=None, -- cgit From a57dffb5fdfbfac59b9ddbe7b33d6f03b7b748ba Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 21 Feb 2011 14:16:42 +0100 Subject: PEP-8 fixes --- nova/utils.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 644bf18fd..5b44bccb5 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -126,7 +126,8 @@ def fetchfile(url, target): execute("curl --fail %s -o %s" % (url, target)) -def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, attempts=1): +def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, + attempts=1): while attempts > 0: attempts -= 1 try: @@ -150,17 +151,16 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, attemp stdout=stdout, stderr=stderr, cmd=cmd) - # NOTE(termie): this appears to be necessary to let the subprocess call - # clean something up in between calls, without it two - # execute calls in a row hangs the second one + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one greenthread.sleep(0) return result except ProcessExecutionError: if not attempts: raise else: - greenthread.sleep(random.randint(50,300)/100) - pass + greenthread.sleep(random.randint(20, 200) / 100.0) def ssh_execute(ssh, cmd, process_input=None, -- cgit From 38c21546ecc079300c575e5950bcb990eecee3a3 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Sun, 27 Feb 2011 20:28:04 -0500 Subject: execute: shell=True removed. --- nova/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 0cf91e0cc..40a8d8d8c 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -125,7 +125,7 @@ def fetchfile(url, target): # c.perform() # c.close() # fp.close() - execute("curl --fail %s -o %s" % (url, target)) + execute("curl","--fail",url,"-o",target) def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): @@ -133,7 +133,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, + obj = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) result = None if process_input != None: @@ -254,7 +254,7 @@ def last_octet(address): def get_my_linklocal(interface): try: - if_str = execute("ip -f inet6 -o addr show %s" % interface) + if_str = execute("ip","-f","inet6","-o","addr","show", interface) condition = "\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link" links = [re.search(condition, x) for x in if_str[0].split('\n')] address = [w.group(1) for w in links if w is not None] -- cgit From 8b3e9ad11c2f5c425701f1eb4abb7b3f577ae1cc Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 28 Feb 2011 12:37:02 +0100 Subject: Add utils.synchronized decorator to allow for synchronising method entrance across multiple workers on the same host. --- nova/utils.py | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 0cf91e0cc..cb1ea5a7d 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -25,6 +25,7 @@ import base64 import datetime import inspect import json +import lockfile import os import random import socket @@ -491,6 +492,16 @@ def loads(s): return json.loads(s) +def synchronized(name): + def wrap(f): + def inner(*args, **kwargs): + lock = lockfile.FileLock('nova-%s.lock' % name) + with lock: + return f(*args, **kwargs) + return inner + return wrap + + def ensure_b64_encoding(val): """Safety method to ensure that values expected to be base64-encoded actually are. If they are, the value is returned unchanged. Otherwise, -- cgit From d5736e925f288462f6325130be0af49f0ace5884 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 28 Feb 2011 23:31:09 +0100 Subject: Add a lock_path flag for lock files. --- nova/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index cb1ea5a7d..48f12350f 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -44,11 +44,13 @@ from eventlet.green import subprocess from nova import exception from nova.exception import ProcessExecutionError +from nova import flags from nova import log as logging LOG = logging.getLogger("nova.utils") TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +FLAGS = flags.FLAGS def import_class(import_str): @@ -495,7 +497,8 @@ def loads(s): def synchronized(name): def wrap(f): def inner(*args, **kwargs): - lock = lockfile.FileLock('nova-%s.lock' % name) + lock = lockfile.FileLock(os.path.join(FLAGS.lock_path, + 'nova-%s.lock' % name)) with lock: return f(*args, **kwargs) return inner -- cgit From be9004ffa4c70358c8edda1f33ffe7ba7e1ae1ee Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Tue, 1 Mar 2011 20:49:46 +0100 Subject: Use functools.wraps to make sure wrapped method's metadata (docstring and name) doesn't get mangled. --- nova/utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 48f12350f..9929e6fef 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -23,11 +23,14 @@ System-level utilities and helper functions. import base64 import datetime +import functools import inspect import json import lockfile +import netaddr import os import random +import re import socket import string import struct @@ -35,8 +38,6 @@ import sys import time import types from xml.sax import saxutils -import re -import netaddr from eventlet import event from eventlet import greenthread @@ -496,6 +497,7 @@ def loads(s): def synchronized(name): def wrap(f): + @functools.wraps(f) def inner(*args, **kwargs): lock = lockfile.FileLock(os.path.join(FLAGS.lock_path, 'nova-%s.lock' % name)) -- cgit From 7b7abe7e7a25c0cd07c64c34f69ce050c669cfc3 Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Mon, 7 Mar 2011 21:54:25 +0100 Subject: Log failed command execution if there are more retry attempts left. --- nova/utils.py | 1 + 1 file changed, 1 insertion(+) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 829adfb9e..80e5b6bbe 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -166,6 +166,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True, if not attempts: raise else: + LOG.debug(_("%r failed. Retrying."), cmd) greenthread.sleep(random.randint(20, 200) / 100.0) -- cgit From cac5881eaa35f94e004c18dd34ca78014f067976 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Tue, 8 Mar 2011 01:01:41 -0500 Subject: execvp --- nova/utils.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 40a8d8d8c..c96b85294 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -125,15 +125,15 @@ def fetchfile(url, target): # c.perform() # c.close() # fp.close() - execute("curl","--fail",url,"-o",target) + execute("curl", "--fail", url, "-o", target) -def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (subprocess): %s"), cmd) +def execute(*cmd, process_input=None, addl_env=None, check_exit_code=True): + LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(cmd, stdin=subprocess.PIPE, + obj = subprocess.Popen(*cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) result = None if process_input != None: @@ -148,7 +148,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, stderr=stderr, - cmd=cmd) + cmd=' '.join(cmd)) # NOTE(termie): this appears to be necessary to let the subprocess call # clean something up in between calls, without it two # execute calls in a row hangs the second one @@ -158,7 +158,7 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): def ssh_execute(ssh, cmd, process_input=None, addl_env=None, check_exit_code=True): - LOG.debug(_("Running cmd (SSH): %s"), cmd) + LOG.debug(_("Running cmd (SSH): %s"), ' '.join(cmd)) if addl_env: raise exception.Error("Environment not supported over SSH") @@ -187,7 +187,7 @@ def ssh_execute(ssh, cmd, process_input=None, raise exception.ProcessExecutionError(exit_code=exit_status, stdout=stdout, stderr=stderr, - cmd=cmd) + cmd=' '.join(cmd)) return (stdout, stderr) @@ -254,7 +254,7 @@ def last_octet(address): def get_my_linklocal(interface): try: - if_str = execute("ip","-f","inet6","-o","addr","show", interface) + if_str = execute("ip", "-f", "inet6", "-o", "addr", "show", interface) condition = "\s+inet6\s+([0-9a-f:]+)/\d+\s+scope\s+link" links = [re.search(condition, x) for x in if_str[0].split('\n')] address = [w.group(1) for w in links if w is not None] -- cgit From a320b5df9f916adf8422ed312306c77570d392c2 Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 00:30:05 -0500 Subject: execvp: almost passes tests --- nova/utils.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index c96b85294..9b51f8b40 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -128,13 +128,20 @@ def fetchfile(url, target): execute("curl", "--fail", url, "-o", target) -def execute(*cmd, process_input=None, addl_env=None, check_exit_code=True): +def execute(*cmd, **kwargs): + process_input=kwargs.get('process_input', None) + addl_env=kwargs.get('addl_env', None) + check_exit_code=kwargs.get('check_exit_code', True) + stdin=kwargs.get('stdin', subprocess.PIPE) + stdout=kwargs.get('stdout', subprocess.PIPE) + stderr=kwargs.get('stderr', subprocess.PIPE) + LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() if addl_env: env.update(addl_env) - obj = subprocess.Popen(*cmd, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env) + obj = subprocess.Popen(cmd, stdin=stdin, + stdout=stdout, stderr=stderr, env=env) result = None if process_input != None: result = obj.communicate(process_input) @@ -220,9 +227,9 @@ def debug(arg): return arg -def runthis(prompt, cmd, check_exit_code=True): - LOG.debug(_("Running %s"), (cmd)) - rv, err = execute(cmd, check_exit_code=check_exit_code) +def runthis(prompt, *cmd, **kwargs): + LOG.debug(_("Running %s"), (" ".join(cmd))) + rv, err = execute(*cmd, **kwargs) def generate_uid(topic, size=8): -- cgit From 1d7358e70379607c9cce02307f4336efbd135a5d Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 01:26:53 -0500 Subject: execvp: unit tests pass --- nova/utils.py | 1 + 1 file changed, 1 insertion(+) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 9b51f8b40..7ddf056ea 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -135,6 +135,7 @@ def execute(*cmd, **kwargs): stdin=kwargs.get('stdin', subprocess.PIPE) stdout=kwargs.get('stdout', subprocess.PIPE) stderr=kwargs.get('stderr', subprocess.PIPE) + cmd=map(str,cmd) LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() -- cgit From 23369a63f4b74fb64bf57554a3fd8b15e3e2b49c Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 14:31:23 -0500 Subject: Fixes uses of process_input --- nova/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 7ddf056ea..0937522ec 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -131,7 +131,7 @@ def fetchfile(url, target): def execute(*cmd, **kwargs): process_input=kwargs.get('process_input', None) addl_env=kwargs.get('addl_env', None) - check_exit_code=kwargs.get('check_exit_code', True) + check_exit_code=kwargs.get('check_exit_code', 0) stdin=kwargs.get('stdin', subprocess.PIPE) stdout=kwargs.get('stdout', subprocess.PIPE) stderr=kwargs.get('stderr', subprocess.PIPE) @@ -151,7 +151,7 @@ def execute(*cmd, **kwargs): obj.stdin.close() if obj.returncode: LOG.debug(_("Result was %s") % obj.returncode) - if check_exit_code and obj.returncode != 0: + if check_exit_code is not None and obj.returncode != check_exit_code: (stdout, stderr) = result raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, -- cgit From fc9840bae6200c8f89fb8a3ba0ab45663c872b3c Mon Sep 17 00:00:00 2001 From: Eric Windisch Date: Wed, 9 Mar 2011 15:33:20 -0500 Subject: execvp passes pep8 --- nova/utils.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 0937522ec..3a4ec3c6a 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -40,7 +40,7 @@ import netaddr from eventlet import event from eventlet import greenthread from eventlet.green import subprocess - +None from nova import exception from nova.exception import ProcessExecutionError from nova import log as logging @@ -129,13 +129,13 @@ def fetchfile(url, target): def execute(*cmd, **kwargs): - process_input=kwargs.get('process_input', None) - addl_env=kwargs.get('addl_env', None) - check_exit_code=kwargs.get('check_exit_code', 0) - stdin=kwargs.get('stdin', subprocess.PIPE) - stdout=kwargs.get('stdout', subprocess.PIPE) - stderr=kwargs.get('stderr', subprocess.PIPE) - cmd=map(str,cmd) + process_input = kwargs.get('process_input', None) + addl_env = kwargs.get('addl_env', None) + check_exit_code = kwargs.get('check_exit_code', 0) + stdin = kwargs.get('stdin', subprocess.PIPE) + stdout = kwargs.get('stdout', subprocess.PIPE) + stderr = kwargs.get('stderr', subprocess.PIPE) + cmd = map(str, cmd) LOG.debug(_("Running cmd (subprocess): %s"), ' '.join(cmd)) env = os.environ.copy() @@ -151,7 +151,8 @@ def execute(*cmd, **kwargs): obj.stdin.close() if obj.returncode: LOG.debug(_("Result was %s") % obj.returncode) - if check_exit_code is not None and obj.returncode != check_exit_code: + if type(check_exit_code) == types.IntType \ + and obj.returncode != check_exit_code: (stdout, stderr) = result raise ProcessExecutionError(exit_code=obj.returncode, stdout=stdout, -- cgit From 11f2d788fd63c66af0e992f7b75b61273c059bcb Mon Sep 17 00:00:00 2001 From: Soren Hansen Date: Thu, 10 Mar 2011 21:31:47 +0100 Subject: PEP8 --- nova/utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'nova/utils.py') diff --git a/nova/utils.py b/nova/utils.py index 3008a512e..87e726394 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -166,9 +166,9 @@ def execute(*cmd, **kwargs): stdout=stdout, stderr=stderr, cmd=' '.join(cmd)) - # NOTE(termie): this appears to be necessary to let the subprocess call - # clean something up in between calls, without it two - # execute calls in a row hangs the second one + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one greenthread.sleep(0) return result except ProcessExecutionError: -- cgit