Differential D826

Get logs from a minion when remote exec crashes
ClosedPublic
Actions

Authored by mkrizek on Apr 25 2016, 4:18 PM.

Details

Reviewers
kparal
tflink
jskladan
Maniphest Tasks
T775: taskotron.log is empty when task execution crashes
Commits
rLTRN9921793aa664: Get logs from a minion when remote exec crashes
Summary

Though, do not try getting the logs when a minion times out, in that
case we could hang forever because the minion could be dead.

Test Plan

Tested locally with both --libvirt and --ssh. Works for me.

Diff Detail

Repository
rLTRN libtaskotron
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.
mkrizek retitled this revision from to Get logs from a minion when remote exec crashes.Apr 25 2016, 4:18 PM
mkrizek updated this object.
mkrizek edited the test plan for this revision. (Show Details)
mkrizek added reviewers: tflink, kparal, jskladan.
tflink accepted this revision.Apr 28 2016, 4:52 AM
This revision is now accepted and ready to land.Apr 28 2016, 4:52 AM
Closed by commit rLTRN9921793aa664: Get logs from a minion when remote exec crashes (authored by mkrizek). · Explain WhyMay 4 2016, 8:41 AM
This revision was automatically updated to reflect the committed changes.

Revision Contents

Diff 2144

View Options

libtaskotron/exceptions.py

Show All 32 Lines31class TaskotronDirectiveError(TaskotronError):
33 pass 33​ pass
34 34​
35 35​
36class TaskotronRemoteError(TaskotronError): 36​class TaskotronRemoteError(TaskotronError):
37 '''All network and remote-server related errors''' 37​ '''All network and remote-server related errors'''
38 pass 38​ pass
39 39​
40 40​
41​class TaskotronRemoteTimeoutError(TaskotronError):
42​ '''A remote-server did not send any output in a given timeout'''
43​ pass
44​
45​
41class TaskotronRemoteProcessError(TaskotronRemoteError): 46​class TaskotronRemoteProcessError(TaskotronRemoteError):
42 '''A process/command executed remotely returned a non-zero exit code, crashed or failed in a 47​ '''A process/command executed remotely returned a non-zero exit code, crashed or failed in a
43 different way''' 48​ different way'''
44 pass 49​ pass
45 50​
46 51​
47class TaskotronNotImplementedError(TaskotronError, NotImplementedError): 52​class TaskotronNotImplementedError(TaskotronError, NotImplementedError):
48 '''NotImplementedError for Taskotron classes, methods and functions''' 53​ '''NotImplementedError for Taskotron classes, methods and functions'''
Show All 25 Lines
View Options

libtaskotron/minion.py

Show First 20 Lines • Show All 154 Lines • ▼ Show 20 Line(s)151 def execute(self):
155 logger.add_filehandler(filelog_path=filelog_path, remove_mem_handler=True) 155​ logger.add_filehandler(filelog_path=filelog_path, remove_mem_handler=True)
156 156​
157 stdio_filename = os.path.join(self.artifactsdir, 'taskotron-stdio.log') 157​ stdio_filename = os.path.join(self.artifactsdir, 'taskotron-stdio.log')
158 158​
159 log.info("Running task over SSH on machine '%s'...", self.machine) 159​ log.info("Running task over SSH on machine '%s'...", self.machine)
160 160​
161 with remote_exec.ParamikoWrapper(self.machine, self.port, self.user, 161​ with remote_exec.ParamikoWrapper(self.machine, self.port, self.user,
162 self.ssh_privkey, stdio_filename) as self.ssh: 162​ self.ssh_privkey, stdio_filename) as self.ssh:
163 self._prepare_task() 163​ try:
164 self._run() 164​ self._prepare_task()
165​ self._run()
166​ except exc.TaskotronRemoteError:
167​ # This is the case when remote execution failed but it should be possible to
168​ # get logs from the minion, so try it. We don't try getting logs in case when
169​ # exc.TaskotronRemoteTimeoutError is thrown which means that the minion is
170​ # likely dead and we would hang on getting the logs.
171​ # See https://phab.qadevel.cloud.fedoraproject.org/T665
172​ self._get_output()
173​ raise
174​
165 self._get_output() 175​ self._get_output()
166 176​
167 177​
178​
168class DisposableMinion(BaseMinion): 179​class DisposableMinion(BaseMinion):
169 '''Minion class that creates a disposable client, connects to it over SSH and runs the task 180​ '''Minion class that creates a disposable client, connects to it over SSH and runs the task
170 there. 181​ there.
171 ''' 182​ '''
172 183​
173 def execute(self): 184​ def execute(self):
174 '''Init environment and connect to disposable client for task execution.''' 185​ '''Init environment and connect to disposable client for task execution.'''
175 186​
Show All 18 Lines193 try:
194 205​
195 log.info('Running task over SSH on machine %s (%s@%s:%s)', task_vm.instancename, 206​ log.info('Running task over SSH on machine %s (%s@%s:%s)', task_vm.instancename,
196 self.user, self.machine, self.port) 207​ self.user, self.machine, self.port)
197 208​
198 task_vm.wait_for_port(self.port) 209​ task_vm.wait_for_port(self.port)
199 210​
200 with remote_exec.ParamikoWrapper(self.machine, self.port, self.user, 211​ with remote_exec.ParamikoWrapper(self.machine, self.port, self.user,
201 self.ssh_privkey, stdio_filename) as self.ssh: 212​ self.ssh_privkey, stdio_filename) as self.ssh:
213​ try:
202 self._prepare_task() 214​ self._prepare_task()
203 self._run() 215​ self._run()
216​ except exc.TaskotronRemoteError:
217​ # This is the case when remote execution failed but it should be possible to
218​ # get logs from the minion, so try it. We don't try getting logs in case when
219​ # exc.TaskotronRemoteTimeoutError is thrown which means that the minion is
220​ # likely dead and we would hang on getting the logs.
221​ # See https://phab.qadevel.cloud.fedoraproject.org/T665
204 self._get_output() 222​ self._get_output()
223​ # propagate the exception into teardown
224​ raise
225​
226​ self._get_output()
227​
205 execution_ok = True 228​ execution_ok = True
206 finally: 229​ finally:
207 if not execution_ok: 230​ if not execution_ok:
208 log.warn('Task execution was interrupted by an error, doing emergency cleanup.') 231​ log.warn('Task execution was interrupted by an error, doing emergency cleanup.')
209 232​
210 if self.arg_data['no_destroy']: 233​ if self.arg_data['no_destroy']:
211 log.info('--no-destroy mode was set from command line, not tearing VM ' 234​ log.info('--no-destroy mode was set from command line, not tearing VM '
212 'down for task %s: %s@%s:%s', self.arg_data['uuid'], self.user, 235​ 'down for task %s: %s@%s:%s', self.arg_data['uuid'], self.user,
Show All 16 Lines
View Options

libtaskotron/remote_exec.py

Show First 20 Lines • Show All 103 Lines • ▼ Show 20 Line(s)26class ParamikoWrapper(object):
104 def cmd(self, cmd): 104​ def cmd(self, cmd):
105 '''Execute a command. 105​ '''Execute a command.
106 106​
107 :param str cmd: A command to be executed. Make sure you escape it properly to prevent shell 107​ :param str cmd: A command to be executed. Make sure you escape it properly to prevent shell
108 expansion, in case it is not desired. 108​ expansion, in case it is not desired.
109 :returns: returncode of the command 109​ :returns: returncode of the command
110 :raise TaskotronRemoteProcessError: If the command has non-zero return code and it isn't a 110​ :raise TaskotronRemoteProcessError: If the command has non-zero return code and it isn't a
111 code of the exitcode directive. 111​ code of the exitcode directive.
112 :raise TaskotronRemoteError: If the remote hasn't sent any output for :attr:`TIMEOUT`. 112​ :raise TaskotronRemoteTimeoutError: If the remote hasn't sent any output for
113​ :attr:`TIMEOUT`.
113 ''' 114​ '''
114 115​
115 log.debug('Running command on remote host: %s', cmd) 116​ log.debug('Running command on remote host: %s', cmd)
116 117​
117 # bufsize=1 means line buffering, which should improve the chance of receiving complete 118​ # bufsize=1 means line buffering, which should improve the chance of receiving complete
118 # lines. get_pty=True allocates a pseudo-terminal, which is needed to have stdout and 119​ # lines. get_pty=True allocates a pseudo-terminal, which is needed to have stdout and
119 # stderr interlaced properly (in the exact chronological order) 120​ # stderr interlaced properly (in the exact chronological order)
120 stdin, stdout, _ = self.ssh.exec_command(cmd, timeout=self.TIMEOUT, bufsize=1, 121​ stdin, stdout, _ = self.ssh.exec_command(cmd, timeout=self.TIMEOUT, bufsize=1,
Show All 11 Lines132 if channel.recv_ready():
132 data = channel.recv(65536) 133​ data = channel.recv(65536)
133 self.outstream.write(data) 134​ self.outstream.write(data)
134 alive_counter = 0 135​ alive_counter = 0
135 136​
136 time.sleep(0.1) 137​ time.sleep(0.1)
137 alive_counter += 0.1 138​ alive_counter += 0.1
138 139​
139 if alive_counter >= self.TIMEOUT: 140​ if alive_counter >= self.TIMEOUT:
140 raise exc.TaskotronRemoteError( 141​ raise exc.TaskotronRemoteTimeoutError(
141 'No output received from machine %s@%s:%s for %d seconds while running: %s' % 142​ 'No output received from machine %s@%s:%s for %d seconds while running: %s' %
142 (self.username, self.hostname, self.port, self.TIMEOUT, cmd)) 143​ (self.username, self.hostname, self.port, self.TIMEOUT, cmd))
143 144​
144 retcode = channel.recv_exit_status() 145​ retcode = channel.recv_exit_status()
145 # reprint the rest of data that was left in the buffer 146​ # reprint the rest of data that was left in the buffer
146 self.outstream.write(stdout.read()) 147​ self.outstream.write(stdout.read())
147 148​
148 if retcode != 0 and retcode != exitcode_directive.FAILURE: 149​ if retcode != 0 and retcode != exitcode_directive.FAILURE:
▲ Show 20 Lines • Show All 193 Lines • Show Last 20 Lines