Differential D586

Use taskotron-disposable copr repo
ClosedPublic
Actions

Authored by mkrizek on Sep 15 2015, 9:12 PM.

Details

Reviewers
lbrabec
kparal
tflink
jskladan
Commits
rLTRN0184b90388d7: Use taskotron-disposable copr repo
Summary

Until the merge is done.

Test Plan

Ran locally. Worked for me.

Diff Detail

Repository
rLTRN libtaskotron
Lint
Automatic diff as part of commit; lint not applicable.
Unit
Automatic diff as part of commit; unit tests not applicable.
mkrizek retitled this revision from to Use taskotron-disposable copr repo.Sep 15 2015, 9:12 PM
mkrizek updated this object.
mkrizek edited the test plan for this revision. (Show Details)
mkrizek added reviewers: tflink, kparal, jskladan, lbrabec.
tflink requested changes to this revision.Sep 15 2015, 9:42 PM

I've been doing this locally for dev/test/demo systems but you do still need the regular taskotron repo as well

libtaskotron/runner.py
347–350

the taskotron-disposable repo isn't enough by itself - you'd still need the regular taskotron repo for some of the deps

This revision now requires changes to proceed.Sep 15 2015, 9:42 PM
mkrizek updated this revision to Diff 1519.Sep 15 2015, 10:07 PM
  • Keep taskotron repo as well
mkrizek updated this revision to Diff 1520.Sep 15 2015, 10:08 PM

Using the right repo now

kparal accepted this revision.Sep 16 2015, 10:10 AM
jskladan accepted this revision.Sep 16 2015, 10:29 AM
tflink accepted this revision.Sep 16 2015, 4:21 PM
This revision is now accepted and ready to land.Sep 16 2015, 4:21 PM
Closed by commit rLTRN0184b90388d7: Use taskotron-disposable copr repo (authored by mkrizek). · Explain WhySep 16 2015, 6:40 PM
This revision was automatically updated to reflect the committed changes.

Revision Contents

PathPackages
Mlibtaskotron/remote_exec.py (25 lines)
Mlibtaskotron/runner.py (11 lines)

Diff 1541

View Options

libtaskotron/remote_exec.py

Show First 20 Lines • Show All 60 Lines • ▼ Show 20 Line(s)48class RemoteExec(object):
61 def __enter__(self): 61​ def __enter__(self):
62 self.connect() 62​ self.connect()
63 return self 63​ return self
64 64​
65 def __exit__(self, type, value, traceback): 65​ def __exit__(self, type, value, traceback):
66 self.close() 66​ self.close()
67 67​
68 def __str__(self): 68​ def __str__(self):
69 return '<%s: %s@%s:%s>' % (self.__class__.__name__, self.username, self.hostname, self.port) 69​ return '<%s: %s@%s:%s>' % (self.__class__.__name__, self.username,
70​ self.hostname, self.port)
70 71​
71 def connect(self): 72​ def connect(self):
72 '''Connect to a machine over ssh. Open sftp channel and, if applicable, file that 73​ '''Connect to a machine over ssh. Open sftp channel and, if applicable, file that
73 stdout/err from the machine will be saved to. 74​ stdout/err from the machine will be saved to.
74 75​
75 :raise TaskotronRemoteError: when the connection does not succeed''' 76​ :raise TaskotronRemoteError: when the connection does not succeed'''
76 77​
77 self.ssh = paramiko.SSHClient() 78​ self.ssh = paramiko.SSHClient()
78 # accept unknown hosts 79​ # accept unknown hosts
79 self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 80​ self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
80 81​
81 log.debug('Connecting to remote host: %s:%s as %s', self.hostname, self.port, self.username) 82​ log.debug('Connecting to remote host: %s@%s:%s', self.username, self.hostname, self.port)
82 83​
83 try: 84​ try:
84 self.ssh.connect(self.hostname, 85​ self.ssh.connect(self.hostname,
85 port=self.port, 86​ port=self.port,
86 username=self.username, 87​ username=self.username,
87 # we use empty string by default (no special key), but paramiko 88​ # we use empty string by default (no special key), but paramiko
88 # requires receiving None in that case 89​ # requires receiving None in that case
89 key_filename=self.key_filename or None) 90​ key_filename=self.key_filename or None)
90 91​
91 self.sftp = self.ssh.open_sftp() 92​ self.sftp = self.ssh.open_sftp()
92 93​
93 if self.stdio_filename is not None: 94​ if self.stdio_filename is not None:
94 try: 95​ try:
95 f = open(self.stdio_filename, 'w') 96​ f = open(self.stdio_filename, 'w')
96 self.outstream.add(f) 97​ self.outstream.add(f)
97 except IOError, e: 98​ except IOError, e:
98 log.warning('Could not open %s. Falling back to writing vm\'s output to stdout only.', 99​ log.warning('Could not open %s. Falling back to writing vm\'s output '
99 self.stdio_filename) 100​ 'to stdout only.', self.stdio_filename)
100 101​
101 except paramiko.BadHostKeyException, e: 102​ except paramiko.BadHostKeyException, e:
102 raise exc.TaskotronRemoteError('Server\'s (%s@%s:%s) hostkey could not be verified: %s' % 103​ raise exc.TaskotronRemoteError('Server\'s (%s@%s:%s) hostkey could not be verified: %s'
103 (self.username, self.hostname, self.port, str(e))) 104​ % (self.username, self.hostname, self.port, str(e)))
104 except paramiko.AuthenticationException, e: 105​ except paramiko.AuthenticationException, e:
105 raise exc.TaskotronRemoteError('Authentication to %s@%s:%s failed: %s' % 106​ raise exc.TaskotronRemoteError('Authentication to %s@%s:%s failed: %s' %
106 (self.username, self.hostname, self.port, str(e))) 107​ (self.username, self.hostname, self.port, str(e)))
107 except (paramiko.SSHException, socket.error), e: 108​ except (paramiko.SSHException, socket.error), e:
108 raise exc.TaskotronRemoteError('Could not connect to %s@%s:%s: %s' % 109​ raise exc.TaskotronRemoteError('Could not connect to %s@%s:%s: %s' %
109 (self.username, self.hostname, self.port, str(e))) 110​ (self.username, self.hostname, self.port, str(e)))
110 except IOError as e: 111​ except IOError as e:
111 # let's hope this does not occur in more situations 112​ # let's hope this does not occur in more situations
▲ Show 20 Lines • Show All 91 Lines • ▼ Show 20 Line(s)203 if self._remote_file_exists(remote_path) and not overwrite:
203 log.info('Remote path %s already exists, not overwriting.', remote_path) 204​ log.info('Remote path %s already exists, not overwriting.', remote_path)
204 return 205​ return
205 206​
206 log.debug('Putting file (%s) on to remote host (%s)', local_path, remote_path) 207​ log.debug('Putting file (%s) on to remote host (%s)', local_path, remote_path)
207 try: 208​ try:
208 return self.sftp.put(local_path, remote_path) 209​ return self.sftp.put(local_path, remote_path)
209 except IOError, e: 210​ except IOError, e:
210 raise exc.TaskotronRemoteError('Could not put file %s to %s@%s:%s: %s' % 211​ raise exc.TaskotronRemoteError('Could not put file %s to %s@%s:%s: %s' %
211 (local_path, self.username, self.hostname, remote_path, e.strerror)) 212​ (local_path, self.username, self.hostname,
213​ remote_path, e.strerror))
212 214​
213 def get_file(self, remote_path, local_path): 215​ def get_file(self, remote_path, local_path):
214 '''Get a file from a remote path. 216​ '''Get a file from a remote path.
215 217​
216 :param str remote_path: A path to the remote file 218​ :param str remote_path: A path to the remote file
217 :param str local_path: A path to the local file 219​ :param str local_path: A path to the local file
218 220​
219 :raise TaskotronRemoteError: If the file could not be downloaded 221​ :raise TaskotronRemoteError: If the file could not be downloaded
220 ''' 222​ '''
221 223​
222 log.debug('Getting file from remote host %s to %s', remote_path, local_path) 224​ log.debug('Getting file from remote host %s to %s', remote_path, local_path)
223 try: 225​ try:
224 self.sftp.get(remote_path, local_path) 226​ self.sftp.get(remote_path, local_path)
225 except IOError, e: 227​ except IOError, e:
226 raise exc.TaskotronRemoteError('Could not copy file %s@%s:%s to %s: %s' % 228​ raise exc.TaskotronRemoteError('Could not copy file %s@%s:%s to %s: %s' %
227 (self.username, self.hostname, remote_path, local_path, e.strerror)) 229​ (self.username, self.hostname, remote_path,
230​ local_path, e.strerror))
228 231​
229 def _remote_file_exists(self, remote_path): 232​ def _remote_file_exists(self, remote_path):
230 try: 233​ try:
231 path_stats = self.sftp.stat(remote_path) 234​ path_stats = self.sftp.stat(remote_path)
232 if path_stats is not None: 235​ if path_stats is not None:
233 return True 236​ return True
234 except IOError: 237​ except IOError:
235 pass 238​ pass
Show All 31 Lines260 def put_dir(self, local_path, remote_path, overwrite=True):
267 if self._remote_file_exists(remote_path) and not overwrite: 270​ if self._remote_file_exists(remote_path) and not overwrite:
268 log.info('Remote path %s already exists, not overwriting.', remote_path) 271​ log.info('Remote path %s already exists, not overwriting.', remote_path)
269 return 272​ return
270 273​
271 try: 274​ try:
272 files = os.listdir(local_path) 275​ files = os.listdir(local_path)
273 except OSError, e: 276​ except OSError, e:
274 raise exc.TaskotronRemoteError('Could not put dir %s to %s@%s:%s: %s' % 277​ raise exc.TaskotronRemoteError('Could not put dir %s to %s@%s:%s: %s' %
275 (local_path, self.username, self.hostname, remote_path, e.strerror)) 278​ (local_path, self.username, self.hostname,
279​ remote_path, e.strerror))
276 280​
277 self.mkdir(remote_path) 281​ self.mkdir(remote_path)
278 282​
279 for file_ in files: 283​ for file_ in files:
280 remote_file = os.path.join(remote_path, file_) 284​ remote_file = os.path.join(remote_path, file_)
281 local_file = os.path.join(local_path, file_) 285​ local_file = os.path.join(local_path, file_)
282 if os.path.isdir(local_file): 286​ if os.path.isdir(local_file):
283 if not self._remote_isdir(remote_file): 287​ if not self._remote_isdir(remote_file):
Show All 10 Lines293 def get_dir(self, remote_path, local_path):
294 298​
295 :raise TaskotronRemoteError: If the directory could not be downloaded 299​ :raise TaskotronRemoteError: If the directory could not be downloaded
296 ''' 300​ '''
297 301​
298 try: 302​ try:
299 files = self.sftp.listdir(remote_path) 303​ files = self.sftp.listdir(remote_path)
300 except IOError, e: 304​ except IOError, e:
301 raise exc.TaskotronRemoteError('Could not copy dir %s@%s:%s to %s: %s' % 305​ raise exc.TaskotronRemoteError('Could not copy dir %s@%s:%s to %s: %s' %
302 (self.username, self.hostname, remote_path, local_path, e.strerror)) 306​ (self.username, self.hostname, remote_path,
307​ local_path, e.strerror))
303 308​
304 for file_ in files: 309​ for file_ in files:
305 remote_file = os.path.join(remote_path, file_) 310​ remote_file = os.path.join(remote_path, file_)
306 local_file = os.path.join(local_path, file_) 311​ local_file = os.path.join(local_path, file_)
307 if self._remote_isdir(remote_file): 312​ if self._remote_isdir(remote_file):
308 os.mkdir(local_file) 313​ os.mkdir(local_file)
309 self.get_dir(remote_file, local_file) 314​ self.get_dir(remote_file, local_file)
310 else: 315​ else:
311 self.get_file(remote_file, local_file) 316​ self.get_file(remote_file, local_file)
View Options

libtaskotron/runner.py

Show First 20 Lines • Show All 286 Lines • ▼ Show 20 Line(s)286 for action in self.task_data['actions']:
287 self.do_single_action(action) 287​ self.do_single_action(action)
288 288​
289 def _validate_input(self): 289​ def _validate_input(self):
290 if 'input' not in self.task_data: 290​ if 'input' not in self.task_data:
291 return 291​ return
292 292​
293 if not isinstance(self.task_data['input'], collections.Mapping): 293​ if not isinstance(self.task_data['input'], collections.Mapping):
294 raise exc.TaskotronYamlError("Input yaml should contain correct 'input'" 294​ raise exc.TaskotronYamlError("Input yaml should contain correct 'input'"
295 "section (a mapping). Yours was: %s" % type( 295​ "section (a mapping). Yours was: %s"
296 self.task_data['input'])) 296​ % type(self.task_data['input']))
297 297​
298 required_args = self.task_data['input'].get('args', None) 298​ required_args = self.task_data['input'].get('args', None)
299 299​
300 if not python_utils.iterable(required_args): 300​ if not python_utils.iterable(required_args):
301 raise exc.TaskotronYamlError("Input yaml should contain correct 'args' " 301​ raise exc.TaskotronYamlError("Input yaml should contain correct 'args' "
302 "section (an iterable). Yours was: %s" % type(required_args)) 302​ "section (an iterable). Yours was: %s" % type(required_args))
303 303​
304 for arg in required_args: 304​ for arg in required_args:
305 if not arg in self.arg_data: 305​ if arg not in self.arg_data:
306 raise exc.TaskotronYamlError("Required input arg '%s' " 306​ raise exc.TaskotronYamlError("Required input arg '%s' "
307 "was not defined" % arg) 307​ "was not defined" % arg)
308 308​
309 def _validate_env(self): 309​ def _validate_env(self):
310 # TODO: implement this 310​ # TODO: implement this
311 raise NotImplementedError("Environment validation is not yet implemented") 311​ raise NotImplementedError("Environment validation is not yet implemented")
312 312​
313 def prepare_task(self): 313​ def prepare_task(self):
314 '''Prepare the environment for running a task. If there are packages, requirements or 314​ '''Prepare the environment for running a task. If there are packages, requirements or
315 groups specified in the task formula, this method tries to install them (in case of 315​ groups specified in the task formula, this method tries to install them (in case of
Show All 23 Lines332 def __init__(self, ssh, task_data, arg_data):
339 self.exitcode = None 339​ self.exitcode = None
340 self.arg_data['artifactsdir'] = os.path.join(config.get_config().artifactsdir, 340​ self.arg_data['artifactsdir'] = os.path.join(config.get_config().artifactsdir,
341 self.arg_data['uuid']) 341​ self.arg_data['uuid'])
342 342​
343 def prepare_task(self): 343​ def prepare_task(self):
344 # FIXME remove this once the vm code is done 344​ # FIXME remove this once the vm code is done
345 self.ssh.cmd('curl -o /etc/yum.repos.d/tflink-taskotron-fedora-f21.repo ' 345​ self.ssh.cmd('curl -o /etc/yum.repos.d/tflink-taskotron-fedora-f21.repo '
346 'http://copr-fe.cloud.fedoraproject.org/coprs/tflink/taskotron/' 346​ 'http://copr-fe.cloud.fedoraproject.org/coprs/tflink/taskotron/'
347 'repo/fedora-21/tflink-taskotron-fedora-21.repo') 347​ 'repo/fedora-21/tflink-taskotron-fedora-21.repo')
348​ self.ssh.cmd('curl -o /etc/yum.repos.d/tflink-taskotron-disposable-fedora-f21.repo '
349​ 'https://copr-fe.cloud.fedoraproject.org/coprs/tflink/taskotron-disposable/'
350​ 'repo/fedora-21/tflink-taskotron-disposable-fedora-21.repo')
tflinkUnsubmitted
Not Done

the taskotron-disposable repo isn't enough by itself - you'd still need the regular taskotron repo for some of the deps

348 self.ssh.install_pkgs('libtaskotron') 351​ self.ssh.install_pkgs('libtaskotron')
349 352​
350 # configure libtaskotron 353​ # configure libtaskotron
351 if config.get_config().config_filename: 354​ if config.get_config().config_filename:
352 self.ssh.put_file(config.get_config().config_filename, '/etc/taskotron/taskotron.yaml') 355​ self.ssh.put_file(config.get_config().config_filename, '/etc/taskotron/taskotron.yaml')
353 356​
354 # create needed dirs 357​ # create needed dirs
355 self.ssh.mkdir(self.taskdir) 358​ self.ssh.mkdir(self.taskdir)
▲ Show 20 Lines • Show All 180 Lines • Show Last 20 Lines