From 6ae66c595d4f85802045734ed1b230a292f9c953 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 20 Aug 2010 13:26:24 +0100 Subject: Better error message on subprocess spawn fail, and it's a ProcessExecutionException irrespective of how the process is run. --- nova/process.py | 65 +++++++++++++++++++++++++++------------------------------ nova/utils.py | 17 +++++++++++++-- 2 files changed, 46 insertions(+), 36 deletions(-) diff --git a/nova/process.py b/nova/process.py index 425d9f162..259e3f92e 100644 --- a/nova/process.py +++ b/nova/process.py @@ -29,28 +29,12 @@ from twisted.internet import protocol from twisted.internet import reactor from nova import flags +from nova.utils import ProcessExecutionError FLAGS = flags.FLAGS flags.DEFINE_integer('process_pool_size', 4, 'Number of processes to use in the process pool') - -# NOTE(termie): this is copied from twisted.internet.utils but since -# they don't export it I've copied and modified -class UnexpectedErrorOutput(IOError): - """ - Standard error data was received where it was not expected. This is a - subclass of L{IOError} to preserve backward compatibility with the previous - error behavior of L{getProcessOutput}. - - @ivar processEnded: A L{Deferred} which will fire when the process which - produced the data on stderr has ended (exited and all file descriptors - closed). - """ - def __init__(self, stdout=None, stderr=None): - IOError.__init__(self, "got stdout: %r\nstderr: %r" % (stdout, stderr)) - - # This is based on _BackRelay from twister.internal.utils, but modified to # capture both stdout and stderr, without odd stderr handling, and also to # handle stdin @@ -62,22 +46,23 @@ class BackRelayWithInput(protocol.ProcessProtocol): @ivar deferred: A L{Deferred} which will be called back with all of stdout and all of stderr as well (as a tuple). C{terminate_on_stderr} is true and any bytes are received over stderr, this will fire with an - L{_UnexpectedErrorOutput} instance and the attribute will be set to + L{_ProcessExecutionError} instance and the attribute will be set to C{None}. @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are received over stderr, this attribute will refer to a L{Deferred} which will be called back when the process ends. This C{Deferred} is also - associated with the L{_UnexpectedErrorOutput} which C{deferred} fires + associated with the L{_ProcessExecutionError} which C{deferred} fires with earlier in this case so that users can determine when the process has actually ended, in addition to knowing when bytes have been received via stderr. """ - def __init__(self, deferred, started_deferred=None, + def __init__(self, deferred, cmd, started_deferred=None, terminate_on_stderr=False, check_exit_code=True, process_input=None): self.deferred = deferred + self.cmd = cmd self.stdout = StringIO.StringIO() self.stderr = StringIO.StringIO() self.started_deferred = started_deferred @@ -85,14 +70,18 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.check_exit_code = check_exit_code self.process_input = process_input self.on_process_ended = None - + + def _build_execution_error(self, exit_code=None): + return ProcessExecutionError( cmd=self.cmd, + exit_code=exit_code, + stdout=self.stdout.getvalue(), + stderr=self.stderr.getvalue()) + def errReceived(self, text): self.stderr.write(text) if self.terminate_on_stderr and (self.deferred is not None): self.on_process_ended = defer.Deferred() - self.deferred.errback(UnexpectedErrorOutput( - stdout=self.stdout.getvalue(), - stderr=self.stderr.getvalue())) + self.deferred.errback(self._build_execution_error()) self.deferred = None self.transport.loseConnection() @@ -102,15 +91,19 @@ class BackRelayWithInput(protocol.ProcessProtocol): def processEnded(self, reason): if self.deferred is not None: stdout, stderr = self.stdout.getvalue(), self.stderr.getvalue() - try: - if self.check_exit_code: - reason.trap(error.ProcessDone) - self.deferred.callback((stdout, stderr)) - except: - # NOTE(justinsb): This logic is a little suspicious to me... - # If the callback throws an exception, then errback will be - # called also. However, this is what the unit tests test for... - self.deferred.errback(UnexpectedErrorOutput(stdout, stderr)) + exit_code = reason.value.exitCode + if self.check_exit_code and exit_code <> 0: + self.deferred.errback(self._build_execution_error(exit_code)) + else: + try: + if self.check_exit_code: + reason.trap(error.ProcessDone) + self.deferred.callback((stdout, stderr)) + except: + # NOTE(justinsb): This logic is a little suspicious to me... + # If the callback throws an exception, then errback will be + # called also. However, this is what the unit tests test for... + self.deferred.errback(_build_execution_error(exit-code)) elif self.on_process_ended is not None: self.on_process_ended.errback(reason) @@ -131,8 +124,12 @@ def get_process_output(executable, args=None, env=None, path=None, args = args and args or () env = env and env and {} deferred = defer.Deferred() + cmd = executable + if args: + cmd = cmd + " " + ' '.join(args) process_handler = BackRelayWithInput( - deferred, + deferred, + cmd, started_deferred=started_deferred, check_exit_code=check_exit_code, process_input=process_input, diff --git a/nova/utils.py b/nova/utils.py index dc3c626ec..b8abb5388 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -36,6 +36,16 @@ from nova import flags FLAGS = flags.FLAGS TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" +class ProcessExecutionError(IOError): + def __init__( self, stdout=None, stderr=None, exit_code=None, cmd=None, + description=None): + if description is None: + description = "Unexpected error while running command." + if exit_code is None: + exit_code = '-' + message = "%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" % ( + description, cmd, exit_code, stdout, stderr) + IOError.__init__(self, message) def import_class(import_str): """Returns a class from a string including module and class""" @@ -73,8 +83,11 @@ def execute(cmd, process_input=None, addl_env=None, check_exit_code=True): if obj.returncode: logging.debug("Result was %s" % (obj.returncode)) if check_exit_code and obj.returncode <> 0: - raise Exception( "Unexpected exit code: %s. result=%s" - % (obj.returncode, result)) + (stdout, stderr) = result + raise ProcessExecutionError(exit_code=obj.returncode, + stdout=stdout, + stderr=stderr, + cmd=cmd) return result -- cgit From 41864e2653286fd46c7b69ee992d4be492b014c6 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 20 Aug 2010 14:50:43 +0100 Subject: Fixed typo --- nova/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/process.py b/nova/process.py index 259e3f92e..81262a506 100644 --- a/nova/process.py +++ b/nova/process.py @@ -103,7 +103,7 @@ class BackRelayWithInput(protocol.ProcessProtocol): # NOTE(justinsb): This logic is a little suspicious to me... # If the callback throws an exception, then errback will be # called also. However, this is what the unit tests test for... - self.deferred.errback(_build_execution_error(exit-code)) + self.deferred.errback(_build_execution_error(exit_code)) elif self.on_process_ended is not None: self.on_process_ended.errback(reason) -- cgit From c4bf107b7e4fd64376dab7ebe39e4531f64879c5 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sat, 21 Aug 2010 11:54:03 +0100 Subject: Added missing "self." --- nova/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nova/process.py b/nova/process.py index 81262a506..069310802 100644 --- a/nova/process.py +++ b/nova/process.py @@ -103,7 +103,7 @@ class BackRelayWithInput(protocol.ProcessProtocol): # NOTE(justinsb): This logic is a little suspicious to me... # If the callback throws an exception, then errback will be # called also. However, this is what the unit tests test for... - self.deferred.errback(_build_execution_error(exit_code)) + self.deferred.errback(self._build_execution_error(exit_code)) elif self.on_process_ended is not None: self.on_process_ended.errback(reason) -- cgit From c3531537aef54b2c27a6e1f28308eac98aec08ba Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 18:32:08 -0700 Subject: whitespace fixes --- nova/process.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/nova/process.py b/nova/process.py index 069310802..259e62358 100644 --- a/nova/process.py +++ b/nova/process.py @@ -35,8 +35,8 @@ FLAGS = flags.FLAGS flags.DEFINE_integer('process_pool_size', 4, 'Number of processes to use in the process pool') -# This is based on _BackRelay from twister.internal.utils, but modified to -# capture both stdout and stderr, without odd stderr handling, and also to +# This is based on _BackRelay from twister.internal.utils, but modified to +# capture both stdout and stderr, without odd stderr handling, and also to # handle stdin class BackRelayWithInput(protocol.ProcessProtocol): """ @@ -46,21 +46,21 @@ class BackRelayWithInput(protocol.ProcessProtocol): @ivar deferred: A L{Deferred} which will be called back with all of stdout and all of stderr as well (as a tuple). C{terminate_on_stderr} is true and any bytes are received over stderr, this will fire with an - L{_ProcessExecutionError} instance and the attribute will be set to + L{_ProcessExecutionError} instance and the attribute will be set to C{None}. - @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are - received over stderr, this attribute will refer to a L{Deferred} which - will be called back when the process ends. This C{Deferred} is also - associated with the L{_ProcessExecutionError} which C{deferred} fires - with earlier in this case so that users can determine when the process + @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are + received over stderr, this attribute will refer to a L{Deferred} which + will be called back when the process ends. This C{Deferred} is also + associated with the L{_ProcessExecutionError} which C{deferred} fires + with earlier in this case so that users can determine when the process has actually ended, in addition to knowing when bytes have been received via stderr. """ - def __init__(self, deferred, cmd, started_deferred=None, - terminate_on_stderr=False, check_exit_code=True, - process_input=None): + def __init__(self, deferred, cmd, started_deferred=None, + terminate_on_stderr=False, check_exit_code=True, + process_input=None): self.deferred = deferred self.cmd = cmd self.stdout = StringIO.StringIO() @@ -70,12 +70,12 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.check_exit_code = check_exit_code self.process_input = process_input self.on_process_ended = None - + def _build_execution_error(self, exit_code=None): - return ProcessExecutionError( cmd=self.cmd, - exit_code=exit_code, - stdout=self.stdout.getvalue(), - stderr=self.stderr.getvalue()) + return ProcessExecutionError(cmd=self.cmd, + exit_code=exit_code, + stdout=self.stdout.getvalue(), + stderr=self.stderr.getvalue()) def errReceived(self, text): self.stderr.write(text) @@ -101,7 +101,7 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.deferred.callback((stdout, stderr)) except: # NOTE(justinsb): This logic is a little suspicious to me... - # If the callback throws an exception, then errback will be + # If the callback throws an exception, then errback will be # called also. However, this is what the unit tests test for... self.deferred.errback(self._build_execution_error(exit_code)) elif self.on_process_ended is not None: @@ -115,8 +115,8 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.transport.write(self.process_input) self.transport.closeStdin() -def get_process_output(executable, args=None, env=None, path=None, - process_reactor=None, check_exit_code=True, +def get_process_output(executable, args=None, env=None, path=None, + process_reactor=None, check_exit_code=True, process_input=None, started_deferred=None, terminate_on_stderr=False): if process_reactor is None: @@ -130,8 +130,8 @@ def get_process_output(executable, args=None, env=None, path=None, process_handler = BackRelayWithInput( deferred, cmd, - started_deferred=started_deferred, - check_exit_code=check_exit_code, + started_deferred=started_deferred, + check_exit_code=check_exit_code, process_input=process_input, terminate_on_stderr=terminate_on_stderr) # NOTE(vish): commands come in as unicode, but self.executes needs @@ -139,7 +139,7 @@ def get_process_output(executable, args=None, env=None, path=None, executable = str(executable) if not args is None: args = [str(x) for x in args] - process_reactor.spawnProcess( process_handler, executable, + process_reactor.spawnProcess( process_handler, executable, (executable,)+tuple(args), env, path) return deferred -- cgit From 6591ac066f1c6f7ca74c540fe5f39033fb41cd10 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 18:32:31 -0700 Subject: one more whitespace fix --- nova/process.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nova/process.py b/nova/process.py index 259e62358..c3b077dc2 100644 --- a/nova/process.py +++ b/nova/process.py @@ -139,8 +139,8 @@ def get_process_output(executable, args=None, env=None, path=None, executable = str(executable) if not args is None: args = [str(x) for x in args] - process_reactor.spawnProcess( process_handler, executable, - (executable,)+tuple(args), env, path) + process_reactor.spawnProcess(process_handler, executable, + (executable,)+tuple(args), env, path) return deferred -- cgit From fc5e1c6f0bee14fdb85ad138324062ceaa598eee Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 21:53:40 -0700 Subject: a few formatting fixes and moved exception --- nova/exception.py | 12 ++++++++++++ nova/process.py | 4 ++-- nova/utils.py | 17 +++++------------ 3 files changed, 19 insertions(+), 14 deletions(-) diff --git a/nova/exception.py b/nova/exception.py index 29bcb17f8..b8894758f 100644 --- a/nova/exception.py +++ b/nova/exception.py @@ -26,6 +26,18 @@ import sys import traceback +class ProcessExecutionError(IOError): + def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, + description=None): + if description is None: + description = "Unexpected error while running command." + if exit_code is None: + exit_code = '-' + message = "%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" % ( + description, cmd, exit_code, stdout, stderr) + IOError.__init__(self, message) + + class Error(Exception): def __init__(self, message=None): super(Error, self).__init__(message) diff --git a/nova/process.py b/nova/process.py index c3b077dc2..5a5d8cbd2 100644 --- a/nova/process.py +++ b/nova/process.py @@ -29,7 +29,7 @@ from twisted.internet import protocol from twisted.internet import reactor from nova import flags -from nova.utils import ProcessExecutionError +from nova.exception import ProcessExecutionError FLAGS = flags.FLAGS flags.DEFINE_integer('process_pool_size', 4, @@ -126,7 +126,7 @@ def get_process_output(executable, args=None, env=None, path=None, deferred = defer.Deferred() cmd = executable if args: - cmd = cmd + " " + ' '.join(args) + cmd = " ".join([cmd] + args) process_handler = BackRelayWithInput( deferred, cmd, diff --git a/nova/utils.py b/nova/utils.py index b8abb5388..d302412ad 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -31,21 +31,12 @@ import sys from nova import exception from nova import flags +from nova.exception import ProcessExecutionError FLAGS = flags.FLAGS TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" -class ProcessExecutionError(IOError): - def __init__( self, stdout=None, stderr=None, exit_code=None, cmd=None, - description=None): - if description is None: - description = "Unexpected error while running command." - if exit_code is None: - exit_code = '-' - message = "%s\nCommand: %s\nExit code: %s\nStdout: %r\nStderr: %r" % ( - description, cmd, exit_code, stdout, stderr) - IOError.__init__(self, message) def import_class(import_str): """Returns a class from a string including module and class""" @@ -118,8 +109,10 @@ def runthis(prompt, cmd, check_exit_code = True): exit_code = subprocess.call(cmd.split(" ")) logging.debug(prompt % (exit_code)) if check_exit_code and exit_code <> 0: - raise Exception( "Unexpected exit code: %s from cmd: %s" - % (exit_code, cmd)) + raise ProcessExecutionError(exit_code=exit_code, + stdout=None, + stderr=None, + cmd=cmd) def generate_uid(topic, size=8): -- cgit