From 6ae66c595d4f85802045734ed1b230a292f9c953 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 20 Aug 2010 13:26:24 +0100 Subject: Better error message on subprocess spawn fail, and it's a ProcessExecutionException irrespective of how the process is run. --- nova/process.py | 65 +++++++++++++++++++++++++++------------------------------ 1 file changed, 31 insertions(+), 34 deletions(-) (limited to 'nova/process.py') diff --git a/nova/process.py b/nova/process.py index 425d9f162..259e3f92e 100644 --- a/nova/process.py +++ b/nova/process.py @@ -29,28 +29,12 @@ from twisted.internet import protocol from twisted.internet import reactor from nova import flags +from nova.utils import ProcessExecutionError FLAGS = flags.FLAGS flags.DEFINE_integer('process_pool_size', 4, 'Number of processes to use in the process pool') - -# NOTE(termie): this is copied from twisted.internet.utils but since -# they don't export it I've copied and modified -class UnexpectedErrorOutput(IOError): - """ - Standard error data was received where it was not expected. This is a - subclass of L{IOError} to preserve backward compatibility with the previous - error behavior of L{getProcessOutput}. - - @ivar processEnded: A L{Deferred} which will fire when the process which - produced the data on stderr has ended (exited and all file descriptors - closed). - """ - def __init__(self, stdout=None, stderr=None): - IOError.__init__(self, "got stdout: %r\nstderr: %r" % (stdout, stderr)) - - # This is based on _BackRelay from twister.internal.utils, but modified to # capture both stdout and stderr, without odd stderr handling, and also to # handle stdin @@ -62,22 +46,23 @@ class BackRelayWithInput(protocol.ProcessProtocol): @ivar deferred: A L{Deferred} which will be called back with all of stdout and all of stderr as well (as a tuple). C{terminate_on_stderr} is true and any bytes are received over stderr, this will fire with an - L{_UnexpectedErrorOutput} instance and the attribute will be set to + L{_ProcessExecutionError} instance and the attribute will be set to C{None}. @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are received over stderr, this attribute will refer to a L{Deferred} which will be called back when the process ends. This C{Deferred} is also - associated with the L{_UnexpectedErrorOutput} which C{deferred} fires + associated with the L{_ProcessExecutionError} which C{deferred} fires with earlier in this case so that users can determine when the process has actually ended, in addition to knowing when bytes have been received via stderr. """ - def __init__(self, deferred, started_deferred=None, + def __init__(self, deferred, cmd, started_deferred=None, terminate_on_stderr=False, check_exit_code=True, process_input=None): self.deferred = deferred + self.cmd = cmd self.stdout = StringIO.StringIO() self.stderr = StringIO.StringIO() self.started_deferred = started_deferred @@ -85,14 +70,18 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.check_exit_code = check_exit_code self.process_input = process_input self.on_process_ended = None - + + def _build_execution_error(self, exit_code=None): + return ProcessExecutionError( cmd=self.cmd, + exit_code=exit_code, + stdout=self.stdout.getvalue(), + stderr=self.stderr.getvalue()) + def errReceived(self, text): self.stderr.write(text) if self.terminate_on_stderr and (self.deferred is not None): self.on_process_ended = defer.Deferred() - self.deferred.errback(UnexpectedErrorOutput( - stdout=self.stdout.getvalue(), - stderr=self.stderr.getvalue())) + self.deferred.errback(self._build_execution_error()) self.deferred = None self.transport.loseConnection() @@ -102,15 +91,19 @@ class BackRelayWithInput(protocol.ProcessProtocol): def processEnded(self, reason): if self.deferred is not None: stdout, stderr = self.stdout.getvalue(), self.stderr.getvalue() - try: - if self.check_exit_code: - reason.trap(error.ProcessDone) - self.deferred.callback((stdout, stderr)) - except: - # NOTE(justinsb): This logic is a little suspicious to me... - # If the callback throws an exception, then errback will be - # called also. However, this is what the unit tests test for... - self.deferred.errback(UnexpectedErrorOutput(stdout, stderr)) + exit_code = reason.value.exitCode + if self.check_exit_code and exit_code <> 0: + self.deferred.errback(self._build_execution_error(exit_code)) + else: + try: + if self.check_exit_code: + reason.trap(error.ProcessDone) + self.deferred.callback((stdout, stderr)) + except: + # NOTE(justinsb): This logic is a little suspicious to me... + # If the callback throws an exception, then errback will be + # called also. However, this is what the unit tests test for... + self.deferred.errback(_build_execution_error(exit-code)) elif self.on_process_ended is not None: self.on_process_ended.errback(reason) @@ -131,8 +124,12 @@ def get_process_output(executable, args=None, env=None, path=None, args = args and args or () env = env and env and {} deferred = defer.Deferred() + cmd = executable + if args: + cmd = cmd + " " + ' '.join(args) process_handler = BackRelayWithInput( - deferred, + deferred, + cmd, started_deferred=started_deferred, check_exit_code=check_exit_code, process_input=process_input, -- cgit From 41864e2653286fd46c7b69ee992d4be492b014c6 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Fri, 20 Aug 2010 14:50:43 +0100 Subject: Fixed typo --- nova/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova/process.py') diff --git a/nova/process.py b/nova/process.py index 259e3f92e..81262a506 100644 --- a/nova/process.py +++ b/nova/process.py @@ -103,7 +103,7 @@ class BackRelayWithInput(protocol.ProcessProtocol): # NOTE(justinsb): This logic is a little suspicious to me... # If the callback throws an exception, then errback will be # called also. However, this is what the unit tests test for... - self.deferred.errback(_build_execution_error(exit-code)) + self.deferred.errback(_build_execution_error(exit_code)) elif self.on_process_ended is not None: self.on_process_ended.errback(reason) -- cgit From c4bf107b7e4fd64376dab7ebe39e4531f64879c5 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Sat, 21 Aug 2010 11:54:03 +0100 Subject: Added missing "self." --- nova/process.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'nova/process.py') diff --git a/nova/process.py b/nova/process.py index 81262a506..069310802 100644 --- a/nova/process.py +++ b/nova/process.py @@ -103,7 +103,7 @@ class BackRelayWithInput(protocol.ProcessProtocol): # NOTE(justinsb): This logic is a little suspicious to me... # If the callback throws an exception, then errback will be # called also. However, this is what the unit tests test for... - self.deferred.errback(_build_execution_error(exit_code)) + self.deferred.errback(self._build_execution_error(exit_code)) elif self.on_process_ended is not None: self.on_process_ended.errback(reason) -- cgit From 9db707dda70bbb11d944ab357841c9bdd5ef5b07 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 05:26:08 -0700 Subject: Lots of fixes to make the nova commands work properly and make datamodel work with mysql properly --- nova/process.py | 95 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 47 insertions(+), 48 deletions(-) (limited to 'nova/process.py') diff --git a/nova/process.py b/nova/process.py index 425d9f162..74725c157 100644 --- a/nova/process.py +++ b/nova/process.py @@ -18,9 +18,10 @@ # under the License. """ -Process pool, still buggy right now. +Process pool using twisted threading """ +import logging import StringIO from twisted.internet import defer @@ -29,30 +30,14 @@ from twisted.internet import protocol from twisted.internet import reactor from nova import flags +from nova.utils import ProcessExecutionError FLAGS = flags.FLAGS flags.DEFINE_integer('process_pool_size', 4, 'Number of processes to use in the process pool') - -# NOTE(termie): this is copied from twisted.internet.utils but since -# they don't export it I've copied and modified -class UnexpectedErrorOutput(IOError): - """ - Standard error data was received where it was not expected. This is a - subclass of L{IOError} to preserve backward compatibility with the previous - error behavior of L{getProcessOutput}. - - @ivar processEnded: A L{Deferred} which will fire when the process which - produced the data on stderr has ended (exited and all file descriptors - closed). - """ - def __init__(self, stdout=None, stderr=None): - IOError.__init__(self, "got stdout: %r\nstderr: %r" % (stdout, stderr)) - - -# This is based on _BackRelay from twister.internal.utils, but modified to -# capture both stdout and stderr, without odd stderr handling, and also to +# This is based on _BackRelay from twister.internal.utils, but modified to +# capture both stdout and stderr, without odd stderr handling, and also to # handle stdin class BackRelayWithInput(protocol.ProcessProtocol): """ @@ -62,22 +47,23 @@ class BackRelayWithInput(protocol.ProcessProtocol): @ivar deferred: A L{Deferred} which will be called back with all of stdout and all of stderr as well (as a tuple). C{terminate_on_stderr} is true and any bytes are received over stderr, this will fire with an - L{_UnexpectedErrorOutput} instance and the attribute will be set to + L{_ProcessExecutionError} instance and the attribute will be set to C{None}. - @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are - received over stderr, this attribute will refer to a L{Deferred} which - will be called back when the process ends. This C{Deferred} is also - associated with the L{_UnexpectedErrorOutput} which C{deferred} fires - with earlier in this case so that users can determine when the process + @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are + received over stderr, this attribute will refer to a L{Deferred} which + will be called back when the process ends. This C{Deferred} is also + associated with the L{_ProcessExecutionError} which C{deferred} fires + with earlier in this case so that users can determine when the process has actually ended, in addition to knowing when bytes have been received via stderr. """ - def __init__(self, deferred, started_deferred=None, - terminate_on_stderr=False, check_exit_code=True, - process_input=None): + def __init__(self, deferred, cmd, started_deferred=None, + terminate_on_stderr=False, check_exit_code=True, + process_input=None): self.deferred = deferred + self.cmd = cmd self.stdout = StringIO.StringIO() self.stderr = StringIO.StringIO() self.started_deferred = started_deferred @@ -85,14 +71,18 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.check_exit_code = check_exit_code self.process_input = process_input self.on_process_ended = None - + + def _build_execution_error(self, exit_code=None): + return ProcessExecutionError(cmd=self.cmd, + exit_code=exit_code, + stdout=self.stdout.getvalue(), + stderr=self.stderr.getvalue()) + def errReceived(self, text): self.stderr.write(text) if self.terminate_on_stderr and (self.deferred is not None): self.on_process_ended = defer.Deferred() - self.deferred.errback(UnexpectedErrorOutput( - stdout=self.stdout.getvalue(), - stderr=self.stderr.getvalue())) + self.deferred.errback(self._build_execution_error()) self.deferred = None self.transport.loseConnection() @@ -102,15 +92,19 @@ class BackRelayWithInput(protocol.ProcessProtocol): def processEnded(self, reason): if self.deferred is not None: stdout, stderr = self.stdout.getvalue(), self.stderr.getvalue() - try: - if self.check_exit_code: - reason.trap(error.ProcessDone) - self.deferred.callback((stdout, stderr)) - except: - # NOTE(justinsb): This logic is a little suspicious to me... - # If the callback throws an exception, then errback will be - # called also. However, this is what the unit tests test for... - self.deferred.errback(UnexpectedErrorOutput(stdout, stderr)) + exit_code = reason.value.exitCode + if self.check_exit_code and exit_code <> 0: + self.deferred.errback(self._build_execution_error(exit_code)) + else: + try: + if self.check_exit_code: + reason.trap(error.ProcessDone) + self.deferred.callback((stdout, stderr)) + except: + # NOTE(justinsb): This logic is a little suspicious to me... + # If the callback throws an exception, then errback will be + # called also. However, this is what the unit tests test for... + self.deferred.errback(self._build_execution_error(exit_code)) elif self.on_process_ended is not None: self.on_process_ended.errback(reason) @@ -122,8 +116,8 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.transport.write(self.process_input) self.transport.closeStdin() -def get_process_output(executable, args=None, env=None, path=None, - process_reactor=None, check_exit_code=True, +def get_process_output(executable, args=None, env=None, path=None, + process_reactor=None, check_exit_code=True, process_input=None, started_deferred=None, terminate_on_stderr=False): if process_reactor is None: @@ -131,10 +125,15 @@ def get_process_output(executable, args=None, env=None, path=None, args = args and args or () env = env and env and {} deferred = defer.Deferred() + cmd = executable + if args: + cmd = cmd + " " + ' '.join(args) + logging.debug("Running cmd: %s", cmd) process_handler = BackRelayWithInput( - deferred, - started_deferred=started_deferred, - check_exit_code=check_exit_code, + deferred, + cmd, + started_deferred=started_deferred, + check_exit_code=check_exit_code, process_input=process_input, terminate_on_stderr=terminate_on_stderr) # NOTE(vish): commands come in as unicode, but self.executes needs @@ -142,7 +141,7 @@ def get_process_output(executable, args=None, env=None, path=None, executable = str(executable) if not args is None: args = [str(x) for x in args] - process_reactor.spawnProcess( process_handler, executable, + process_reactor.spawnProcess( process_handler, executable, (executable,)+tuple(args), env, path) return deferred -- cgit From c3531537aef54b2c27a6e1f28308eac98aec08ba Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 18:32:08 -0700 Subject: whitespace fixes --- nova/process.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) (limited to 'nova/process.py') diff --git a/nova/process.py b/nova/process.py index 069310802..259e62358 100644 --- a/nova/process.py +++ b/nova/process.py @@ -35,8 +35,8 @@ FLAGS = flags.FLAGS flags.DEFINE_integer('process_pool_size', 4, 'Number of processes to use in the process pool') -# This is based on _BackRelay from twister.internal.utils, but modified to -# capture both stdout and stderr, without odd stderr handling, and also to +# This is based on _BackRelay from twister.internal.utils, but modified to +# capture both stdout and stderr, without odd stderr handling, and also to # handle stdin class BackRelayWithInput(protocol.ProcessProtocol): """ @@ -46,21 +46,21 @@ class BackRelayWithInput(protocol.ProcessProtocol): @ivar deferred: A L{Deferred} which will be called back with all of stdout and all of stderr as well (as a tuple). C{terminate_on_stderr} is true and any bytes are received over stderr, this will fire with an - L{_ProcessExecutionError} instance and the attribute will be set to + L{_ProcessExecutionError} instance and the attribute will be set to C{None}. - @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are - received over stderr, this attribute will refer to a L{Deferred} which - will be called back when the process ends. This C{Deferred} is also - associated with the L{_ProcessExecutionError} which C{deferred} fires - with earlier in this case so that users can determine when the process + @ivar onProcessEnded: If C{terminate_on_stderr} is false and bytes are + received over stderr, this attribute will refer to a L{Deferred} which + will be called back when the process ends. This C{Deferred} is also + associated with the L{_ProcessExecutionError} which C{deferred} fires + with earlier in this case so that users can determine when the process has actually ended, in addition to knowing when bytes have been received via stderr. """ - def __init__(self, deferred, cmd, started_deferred=None, - terminate_on_stderr=False, check_exit_code=True, - process_input=None): + def __init__(self, deferred, cmd, started_deferred=None, + terminate_on_stderr=False, check_exit_code=True, + process_input=None): self.deferred = deferred self.cmd = cmd self.stdout = StringIO.StringIO() @@ -70,12 +70,12 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.check_exit_code = check_exit_code self.process_input = process_input self.on_process_ended = None - + def _build_execution_error(self, exit_code=None): - return ProcessExecutionError( cmd=self.cmd, - exit_code=exit_code, - stdout=self.stdout.getvalue(), - stderr=self.stderr.getvalue()) + return ProcessExecutionError(cmd=self.cmd, + exit_code=exit_code, + stdout=self.stdout.getvalue(), + stderr=self.stderr.getvalue()) def errReceived(self, text): self.stderr.write(text) @@ -101,7 +101,7 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.deferred.callback((stdout, stderr)) except: # NOTE(justinsb): This logic is a little suspicious to me... - # If the callback throws an exception, then errback will be + # If the callback throws an exception, then errback will be # called also. However, this is what the unit tests test for... self.deferred.errback(self._build_execution_error(exit_code)) elif self.on_process_ended is not None: @@ -115,8 +115,8 @@ class BackRelayWithInput(protocol.ProcessProtocol): self.transport.write(self.process_input) self.transport.closeStdin() -def get_process_output(executable, args=None, env=None, path=None, - process_reactor=None, check_exit_code=True, +def get_process_output(executable, args=None, env=None, path=None, + process_reactor=None, check_exit_code=True, process_input=None, started_deferred=None, terminate_on_stderr=False): if process_reactor is None: @@ -130,8 +130,8 @@ def get_process_output(executable, args=None, env=None, path=None, process_handler = BackRelayWithInput( deferred, cmd, - started_deferred=started_deferred, - check_exit_code=check_exit_code, + started_deferred=started_deferred, + check_exit_code=check_exit_code, process_input=process_input, terminate_on_stderr=terminate_on_stderr) # NOTE(vish): commands come in as unicode, but self.executes needs @@ -139,7 +139,7 @@ def get_process_output(executable, args=None, env=None, path=None, executable = str(executable) if not args is None: args = [str(x) for x in args] - process_reactor.spawnProcess( process_handler, executable, + process_reactor.spawnProcess( process_handler, executable, (executable,)+tuple(args), env, path) return deferred -- cgit From 6591ac066f1c6f7ca74c540fe5f39033fb41cd10 Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 18:32:31 -0700 Subject: one more whitespace fix --- nova/process.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'nova/process.py') diff --git a/nova/process.py b/nova/process.py index 259e62358..c3b077dc2 100644 --- a/nova/process.py +++ b/nova/process.py @@ -139,8 +139,8 @@ def get_process_output(executable, args=None, env=None, path=None, executable = str(executable) if not args is None: args = [str(x) for x in args] - process_reactor.spawnProcess( process_handler, executable, - (executable,)+tuple(args), env, path) + process_reactor.spawnProcess(process_handler, executable, + (executable,)+tuple(args), env, path) return deferred -- cgit From fc5e1c6f0bee14fdb85ad138324062ceaa598eee Mon Sep 17 00:00:00 2001 From: Vishvananda Ishaya Date: Tue, 7 Sep 2010 21:53:40 -0700 Subject: a few formatting fixes and moved exception --- nova/process.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'nova/process.py') diff --git a/nova/process.py b/nova/process.py index c3b077dc2..5a5d8cbd2 100644 --- a/nova/process.py +++ b/nova/process.py @@ -29,7 +29,7 @@ from twisted.internet import protocol from twisted.internet import reactor from nova import flags -from nova.utils import ProcessExecutionError +from nova.exception import ProcessExecutionError FLAGS = flags.FLAGS flags.DEFINE_integer('process_pool_size', 4, @@ -126,7 +126,7 @@ def get_process_output(executable, args=None, env=None, path=None, deferred = defer.Deferred() cmd = executable if args: - cmd = cmd + " " + ' '.join(args) + cmd = " ".join([cmd] + args) process_handler = BackRelayWithInput( deferred, cmd, -- cgit