1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
|
# fedpkg - a Python library for Fedora Packagers
#
# Copyright (C) 2009 Red Hat Inc.
# Author(s): Jesse Keating <jkeating@redhat.com>
#
# This program is free software; you can redistribute it and/or modify it
# under the terms of the GNU General Public License as published by the
# Free Software Foundation; either version 2 of the License, or (at your
# option) any later version. See http://www.gnu.org/copyleft/gpl.html for
# the full text of the license.
import os
#import pycurl
import subprocess
import hashlib
import koji
import rpm
import logging
# Define some global variables, put them here to make it easy to change
LOOKASIDE = 'http://cvs.fedoraproject.org/repo/pkgs'
LOOKASIDEHASH = 'md5'
GITBASEURL = 'ssh://%(user)s@pkgs.stg.fedoraproject.org/%(module)s'
# Define our own error class
class FedpkgError(Exception):
pass
# Setup our logger
# Null logger to avoid spurrious messages, add a handler in app code
class NullHandler(logging.Handler):
def emit(self, record):
pass
h = NullHandler()
# This is our log object, clients of this library can use this object to
# define their own logging needs
log = logging.getLogger("fedpkg")
# Add the null handler
log.addHandler(h)
# Define some helper functions, they start with _
def _hash_file(file, hashtype):
"""Return the hash of a file given a hash type"""
try:
sum = hashlib.new(hashtype)
except ValueError:
raise FedpkgError('Invalid hash type: %s' % hashtype)
input = open(file, 'rb')
# Loop through the file reading chunks at a time as to not
# put the entire file in memory. That would suck for DVDs
while True:
chunk = input.read(8192) # magic number! Taking suggestions
if not chunk:
break # we're done with the file
sum.update(chunk)
input.close()
return sum.hexdigest()
def _verify_file(file, hash, hashtype):
"""Given a file, a hash of that file, and a hashtype, verify.
Returns True if the file verifies, False otherwise
"""
# get the hash
sum = _hash_file(file, hashtype)
# now do the comparison
if sum == hash:
return True
return False
def _get_build_arches_from_srpm(srpm, arches):
"""Given the path to an srpm, determine the possible build arches
Use supplied arches as a filter, only return compatible arches
"""
archlist = arches
hdr = koji.get_rpm_header(srpm)
if hdr[rpm.RPMTAG_SOURCEPACKAGE] != 1:
raise FedpkgError('%s is not a source package.' % srpm)
buildarchs = hdr[rpm.RPMTAG_BUILDARCHS]
exclusivearch = hdr[rpm.RPMTAG_EXCLUSIVEARCH]
excludearch = hdr[rpm.RPMTAG_EXCLUDEARCH]
# Reduce by buildarchs
if buildarchs:
archlist = [a for a in archlist if a in buildarchs]
# Reduce by exclusive arches
if exclusivearch:
archlist = [a for a in archlist if a in exclusivearch]
# Reduce by exclude arch
if excludearch:
archlist = [a for a in archlist if a not in excludearch]
# do the noarch thing
if 'noarch' not in excludearch and ('noarch' in buildarchs or \
'noarch' in exclusivearch):
archlist.append('noarch')
# See if we have anything compatible. Should we raise here?
if not archlist:
raise FedpkgError('No compatible build arches found in %s' % srpm)
return archlist
def clean(dry=False, useignore=True):
"""Clean a module checkout of untracked files.
Can optionally perform a dry-run
Can optionally not use the ignore rules
Returns output
"""
# setup the command, this could probably be done with some python api...
cmd = ['git', 'clean', '-f', '-d']
if dry:
cmd.append('--dry-run')
if not useignore:
cmd.append('-x')
# Run it!
try:
proc = subprocess.Popen(cmd, stderr=subprocess.STDOUT,
stdout=subprocess.PIPE)
output = proc.communicate()
except OSError, e:
raise FedpkgError(e)
# See if we exited cleanly
if proc.returncode:
raise FedpkgError('%s returned %s: %s' %
(subprocess.list2cmdline(cmd),
proc.returncode, output[0]))
return output[0]
def clone(module, user, branch=None):
"""Clone a repo, optionally check out a specific branch.
module is the name of the module to clone
branch is the name of a branch to checkout instead of origin/master
gitargs is an optional list of arguments to git clone
"""
# not implemented yet
# construct the git url
giturl = GITBASEURL % {'user': user, 'module': module}
cmd = ['git', 'clone']
if branch:
cmd.extend(['--branch', branch])
cmd.append(giturl)
print('Would have ran %s' % subprocess.list2cmdline(cmd))
return
def clone_with_dirs(module, user):
"""Clone a repo old style with subdirs for each branch.
module is the name of the module to clone
gitargs is an option list of arguments to git clone
"""
# not implemented yet
print('would have cloned %s with dirs as user %s' %
(module, user))
return
# Create a class for package module
class PackageModule:
def _findbranch(self):
"""Find the branch we're on"""
if not os.path.exists(os.path.join(self.path, 'branch')):
return 'devel'
branch = open(os.path.join(self.path, 'branch'), 'r').read().strip()
return branch
def _getlocalarch(self):
"""Get the local arch as defined by rpm"""
return subprocess.Popen(['rpm --eval %{_arch}'], shell=True,
stdout=subprocess.PIPE).communicate()[0].strip('\n')
def __init__(self, path=os.getcwd()):
# Initiate a PackageModule object in a given path
# Set some global variables used throughout
self.path = path
self.lookaside = LOOKASIDE
self.lookasidehash = LOOKASIDEHASH
self.spec = self.gimmespec()
self.module = self.spec.split('.spec')[0]
# Find the branch and set things based from that
# Still requires a 'branch' file in each branch
self.branch = self._findbranch()
if self.branch.startswith('F-'):
self.distval = self.branch.split('-')[1]
self.distvar = 'fedora'
self.dist = '.fc%s' % self.distval
elif self.branch.startswith('EL-'):
self.distval = self.branch.split('-')[1]
self.distvar = 'epel'
self.dist = '.el%s' % self.distval
elif self.branch.startswith('OLPC-'):
self.distval = self.branch.split('-')[1]
self.distvar = 'olpc'
self.dist = '.olpc%s' % self.distval
# Need to do something about no branch here
elif self.branch == 'devel':
self.distval = '13' # this is hardset for now, which is bad
self.distvar = 'fedora'
self.dist = '.fc%s' % self.distval
self.rpmdefines = ["--define '_sourcedir %s'" % path,
"--define '_specdir %s'" % path,
"--define '_builddir %s'" % path,
"--define '_srcrpmdir %s'" % path,
"--define '_rpmdir %s'" % path,
"--define 'dist %s'" % self.dist,
"--define '%s %s'" % (self.distvar, self.distval),
"--define '%s 1'" % self.distvar]
self.ver = self.getver()
self.rel = self.getrel()
self.localarch = self._getlocalarch()
def compile(self, arch=None, short=False):
"""Run rpm -bc on a module
optionally for a specific arch, or short-circuit it
Returns the output"""
# Get the sources
self.sources()
# setup the rpm command
cmd = ['rpmbuild']
cmd.extend(self.rpmdefines)
if arch:
cmd.extend(['--target', arch])
if short:
cmd.append('--short-circuit')
cmd.extend(['-bc', os.path.join(self.path, self.spec)])
# Run the command and capture output
try:
proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.STDOUT,
stdout=subprocess.PIPE, shell=True)
output = proc.communicate()
except OSError, e:
raise FedpkgError(e)
# See if we exited cleanly
if proc.returncode:
raise FedpkgError('%s returned %s: %s' %
(subprocess.list2cmdline(cmd),
proc.returncode, output[0]))
return output[0]
def getver(self):
"""Return the version-release of a package module."""
cmd = ['rpm']
cmd.extend(self.rpmdefines)
cmd.extend(['-q', '--qf', '%{VERSION}', '--specfile',
os.path.join(self.path, self.spec)])
try:
output = subprocess.Popen(' '.join(cmd), shell=True,
stdout=subprocess.PIPE).communicate()
except subprocess.CalledProcessError, e:
raise FedpkgError('Could not get version of %s: %s' % (self.module, e))
return output[0]
def getrel(self):
"""Return the version-release of a package module."""
cmd = ['rpm']
cmd.extend(self.rpmdefines)
cmd.extend(['-q', '--qf', '%{RELEASE}', '--specfile',
os.path.join(self.path, self.spec)])
try:
output = subprocess.Popen(' '.join(cmd), shell=True,
stdout=subprocess.PIPE).communicate()
except subprocess.CalledProcessError, e:
raise FedpkgError('Could not get release of %s: %s' % (self.module, e))
return output[0]
def gimmespec(self):
"""Print the name of a specfile within a package module"""
# Get a list of files in the path we're looking at
files = os.listdir(self.path)
# Search the files for the first one that ends with ".spec"
for f in files:
if f.endswith('.spec'):
return f
raise FedpkgError('No spec file found.')
def install(self, arch=None, short=False):
"""Run rpm -bi on a module
optionally for a specific arch, or short-circuit it
Returns the output"""
# Get the sources
self.sources()
# setup the rpm command
cmd = ['rpmbuild']
cmd.extend(self.rpmdefines)
if arch:
cmd.extend(['--target', arch])
if short:
cmd.append('--short-circuit')
cmd.extend(['-bi', os.path.join(self.path, self.spec)])
# Run the command and capture output
try:
proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.STDOUT,
stdout=subprocess.PIPE, shell=True)
output = proc.communicate()
except OSError, e:
raise FedpkgError(e)
# See if we exited cleanly
if proc.returncode:
raise FedpkgError('%s returned %s: %s' %
(subprocess.list2cmdline(cmd),
proc.returncode, output[0]))
return output[0]
def lint(self):
"""Run rpmlint over a built srpm"""
# Make sure we have rpms to run on
srpm = "%s-%s-%s.src.rpm" % (self.module, self.ver, self.rel)
if not os.path.exists(os.path.join(self.path, srpm)):
raise FedpkgError('Need to build srpm and rpm first')
# Get the possible built arches
arches = _get_build_arches_from_srpm(os.path.join(self.path, srpm),
[self.localarch])
rpms = []
for arch in arches:
rpms.extend([os.path.join(self.path, arch, file) for file in
os.listdir(os.path.join(self.path, arch))
if file.endswith('.rpm')])
cmd = ['rpmlint', os.path.join(self.path, srpm)]
cmd.extend(rpms)
try:
output = subprocess.Popen(cmd, stdout=subprocess.PIPE).communicate()
except subprocess.CalledProcessError, e:
raise FedpkgError(e)
return output[0]
def local(self, arch=None, hashtype='sha256'):
"""rpmbuild locally for given arch.
Takes arch to build for, and hashtype to build with.
Writes output to a log file and returns output from build.
"""
# This could really use a list of arches to build for and loop over
# Get the sources
self.sources()
# Determine arch to build for
if not arch:
arch = self.localarch
# build up the rpm command
cmd = ['rpmbuild']
cmd.extend(self.rpmdefines)
# This may need to get updated if we ever change our checksum default
if not hashtype == 'sha256':
cmd.extend(["--define '_source_filedigest_algorithm %s'" % hashtype,
"--define '_binary_filedigest_algorithm %s'" % hashtype])
cmd.extend(['--target', arch, '-ba',
os.path.join(self.path, self.spec)])
try:
proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.STDOUT,
stdout=subprocess.PIPE, shell=True)
output = proc.communicate()
except OSError, e:
raise FedpkgError(e)
outfile = open(os.path.join(self.path, '.build-%s-%s.log' % (self.ver,
self.rel)), 'w')
outfile.writelines(output[0])
outfile.close()
# See if we had a good return or not, raise accordingly
if proc.returncode:
raise FedpkgError('%s returned %s: %s' %
(subprocess.list2cmdline(cmd),
proc.returncode, output[0]))
return output[0]
def new_sources(self, files):
"""Replace source file(s) in the lookaside cache"""
# Not fully implimented yet
for file in files:
hash = _hash_file(file, self.lookasidehash)
print "Would upload %s:%s" % (hash, file)
return
def prep(self, arch=None):
"""Run rpm -bp on a module
optionally for a specific arch
Returns the output"""
# Get the sources
self.sources()
# setup the rpm command
cmd = ['rpmbuild']
cmd.extend(self.rpmdefines)
if arch:
cmd.extend(['--target', arch])
cmd.extend(['--nodeps', '-bp', os.path.join(self.path, self.spec)])
# Run the command and capture output
try:
proc = subprocess.Popen(' '.join(cmd), stderr=subprocess.STDOUT,
stdout=subprocess.PIPE, shell=True)
output = proc.communicate()
except OSError, e:
raise FedpkgError(e)
# See if we exited cleanly
if proc.returncode:
raise FedpkgError('%s returned %s: %s' %
(subprocess.list2cmdline(cmd),
proc.returncode, output[0]))
return output[0]
def sources(self, outdir=None):
"""Download source files"""
archives = open(os.path.join(self.path, 'sources'),
'r').readlines()
# Default to putting the files where the module is
if not outdir:
outdir = self.path
for archive in archives:
csum, file = archive.split()
# See if we already have a valid copy downloaded
outfile = os.path.join(outdir, file)
if os.path.exists(outfile):
if _verify_file(outfile, csum, self.lookasidehash):
continue
url = '%s/%s/%s/%s/%s' % (self.lookaside, self.module, file, csum,
file)
# There is some code here for using pycurl, but for now,
# just use subprocess
#output = open(file, 'wb')
#curl = pycurl.Curl()
#curl.setopt(pycurl.URL, url)
#curl.setopt(pycurl.FOLLOWLOCATION, 1)
#curl.setopt(pycurl.MAXREDIRS, 5)
#curl.setopt(pycurl.CONNECTTIMEOUT, 30)
#curl.setopt(pycurl.TIMEOUT, 300)
#curl.setopt(pycurl.WRITEDATA, output)
#try:
# curl.perform()
#except:
# print "Problems downloading %s" % url
# curl.close()
# output.close()
# return 1
#curl.close()
#output.close()
# These options came from Makefile.common.
# Probably need to support wget too
command = ['curl', '-H', 'Pragma:', '-O', '-R', '-S', '--fail',
'--show-error', url]
try:
subprocess.check_call(command, cwd=outdir)
except subprocess.CalledProcessError, e:
raise FedpkgError('Could not download %s: %s' % (url, e))
if not _verify_file(outfile, csum, self.lookasidehash):
raise FedpkgError('%s failed checksum' % file)
return
def srpm(self, hashtype='sha256'):
"""Create an srpm using hashtype from content in the module
Requires sources already downloaded.
"""
cmd = ['rpmbuild']
cmd.extend(self.rpmdefines)
# This may need to get updated if we ever change our checksum default
if not hashtype == 'sha256':
cmd.extend(["--define '_source_filedigest_algorithm %s'" % hashtype,
"--define '_binary_filedigest_algorithm %s'" % hashtype])
cmd.extend(['--nodeps', '-bs', os.path.join(self.path, self.spec)])
try:
subprocess.check_call(' '.join(cmd), shell=True)
except subprocess.CalledProcessError, e:
raise FedpkgError('Could not build %s: %s' % (self.module, e))
return
|