diff options
Diffstat (limited to 'createprestorepo/gendeltarpms.py')
| -rwxr-xr-x | createprestorepo/gendeltarpms.py | 275 |
1 files changed, 122 insertions, 153 deletions
diff --git a/createprestorepo/gendeltarpms.py b/createprestorepo/gendeltarpms.py index 5e0226f..fc4a9ca 100755 --- a/createprestorepo/gendeltarpms.py +++ b/createprestorepo/gendeltarpms.py @@ -20,17 +20,77 @@ import fnmatch, re import rpmUtils.transaction, rpmUtils.miscutils import commands, libxml2 import string +from dumpMetadata import byteranges +from packagelist import RpmItem, DrpmItem, PackageList -SUFFIX='drpm' -DRPMWORTHKEEPINGTHRESH=0.5 -DEBUG=1 +import gzip +from zlib import error as zlibError +from gzip import write32u, FNAME -def getFileList(basepath, path, ext, filelist): + +DRPMWORTHKEEPINGTHRESH=0.7 +DEBUG=0 + +def getDeltaNevr(filename): + """Get nevr for src rpm of a deltarpm. Return a tuple of (n, e, v, r)""" + def _getLength(in_data): + length = 0 + for val in in_data: + length = length * 256 + length += ord(val) + return length + + def _stringToVersion(strng): + i = strng.find(':') + if i != -1: + epoch = strng[:i] + else: + epoch = '0' + j = strng.find('-') + if j != -1: + if strng[i + 1:j] == '': + version = None + else: + version = strng[i + 1:j] + release = strng[j + 1:] + else: + if strng[i + 1:] == '': + version = None + else: + version = strng[i + 1:] + release = None + return (epoch, version, release) + + def _stringToNEVR(string): + i = string.rfind("-", 0, string.rfind("-")-1) + name = string[:i] + (epoch, ver, rel) = _stringToVersion(string[i+1:]) + return (name, epoch, ver, rel) + + (range_start, range_end) = byteranges(filename) + fd = open(filename, "r") + fd.seek(range_end) + try: + compobj = gzip.GzipFile("", "rb", 9, fd) + except: + raise zlibError("Data not stored in gzip format") + + if compobj.read(4)[:3] != "DLT": + raise Exception("Not a deltarpm") + + nevr_length = _getLength(compobj.read(4)) + nevr = compobj.read(nevr_length).strip("\x00") + oldnevr = _stringToNEVR(nevr) + compobj.close() + return oldnevr + + +def getFileList(path, ext, filelist): """Return all files in path matching ext, store them in filelist, recurse dirs. Returns a list object""" extlen = len(ext) - totalpath = os.path.normpath(os.path.join(basepath, path)) + totalpath = os.path.normpath(path) try: dir_list = os.listdir(totalpath) except OSError, e: @@ -39,159 +99,76 @@ def getFileList(basepath, path, ext, filelist): for d in dir_list: if os.path.isdir(totalpath + '/' + d): - filelist = getFileList(basepath, os.path.join(path, d), ext, filelist) - else: - if string.lower(d[-extlen:]) == '%s' % (ext): - if totalpath.find(basepath) == 0: - if basepath != "": - relativepath = totalpath.replace(basepath, "", 1) - else: - relativepath = totalpath - relativepath = relativepath.lstrip("/") - filelist.append(os.path.join(relativepath, d)) - else: - raise "basepath '%s' not found in path '%s'" % (basepath, totalpath) + filelist = getFileList(os.path.join(totalpath, d), ext, filelist) + elif string.lower(d[-extlen:]) == '%s' % (ext): + filelist.append(os.path.join(totalpath, d)) return filelist - -def genDeltaRPM(ts, newrpm, oldrpm, srcdir, dstdir, old_delta = None): - (f1,n1,e1,v1,r1,a1) = newrpm - (f2,n2,e2,v2,r2,a2) = oldrpm - hdr = rpmUtils.miscutils.hdrFromPackage(ts,f1) - arch = hdr['arch'] - v12 = "_".join([v1,v2]) - r12 = "_".join([r1,r2]) - deltaRPMName= '%s/%s.%s.%s' % (dstdir, "-".join([n1,v12,r12]), a1, SUFFIX) - - # If the drpm doesn't exist, make it, else skip it - if os.path.exists("%s%s" % (srcdir, deltaRPMName)) or os.path.exists("%s%s.dontdelta" % (srcdir, deltaRPMName)): -# if e1 == e2: -# print 'Using pre-generated delta rpm for %s.%s - %s.%s => %s.%s' % (n1, a1, v2, r2, v1, r1) -# else: -# print 'Using pre-generated delta rpm for %s.%s - %s:%s.%s => %s:%s.%s' % (n1, a1, e2, v2, r2, e1, v1, r1) - if DEBUG: - print "DEBUG skipping %s" % (deltaRPMName) - else: - generated = False - prep_made = False - if old_delta != None: - (df,dn,de,dv,dr,da) = old_delta - v1d = "_".join([v1,dv]) - r1d = "_".join([r1,dr]) - src_deltaRPMFile= '%s%s/%s.%s.%s' % (srcdir, dstdir, "-".join([n1,v1d,r1d]), a1, SUFFIX) - vd2 = "_".join([dv,v2]) - rd2 = "_".join([dr,r2]) - old_deltaRPMFile= '%s%s/%s.%s.%s' % (srcdir, dstdir, "-".join([n1,vd2,rd2]), a1, SUFFIX) - if os.access(old_deltaRPMFile, os.R_OK) and os.access(src_deltaRPMFile, os.R_OK): - deltaCommand = 'combinedeltarpm %s %s %s%s' % (old_deltaRPMFile, src_deltaRPMFile, srcdir, deltaRPMName) - if DEBUG: - print deltaCommand - prep_made = True - - if not prep_made: - deltaCommand = 'makedeltarpm %s %s %s%s' % (f2, f1, srcdir, deltaRPMName) - if DEBUG: - print deltaCommand - - (code, out) = commands.getstatusoutput(deltaCommand) - if code: - #raise Exception("genDeltaRPM: exitcode was %s - Reported Error: %s" % (code, out)) - print "Error genDeltaRPM for %s: exitcode was %s - Reported Error: %s" % (n1, code, out) - else: - # Check whether or not we should keep the drpm - if not drpmIsWorthKeeping(deltaRPMName, f1, srcdir): - if DEBUG: - print 'deleting %s' % (deltaRPMName) - try: - os.unlink("%s%s" % (srcdir, deltaRPMName)) - except Exception, e: - print "Error deleting deltarpm %s" % (deltaRPMName), str(e) - f = open("%s%s.dontdelta" % (srcdir, deltaRPMName), "w") - f.close() - else: - if e1 == e2: - print 'Generated delta rpm for %s.%s - %s-%s => %s-%s' % (n1, a1, v2, r2, v1, r1) - else: - print 'Generated delta rpm for %s.%s - %s:%s-%s => %s:%s-%s' % (n1, a1, e2, v2, r2, e1, v1, r1) - return deltaRPMName - -def drpmIsWorthKeeping(deltaRPMName, newrpm, srcdir): - newsize = os.path.getsize(newrpm) - drpmsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName)) - # Delete the drpm if it's too large - if drpmsize > DRPMWORTHKEEPINGTHRESH * newsize: - return False - return True - -def createPrestoRepo(srcdir, dstdir): +def createPrestoRepo(base_dir, drpm_dir): ts = rpmUtils.transaction.initReadOnlyTransaction() changed = False # Create list of .rpm files. # We don't use "glob", so sub-directories are supported. - print 'Using source dir: %s' % srcdir - print 'Using destination dir: %s' % dstdir - if dstdir[-1] == "/": - dstdir = dstdir[:-1] - srcfiles = [] - srcfiles = getFileList("", srcdir, ".rpm", []) - if not len(srcfiles): + print 'Using base dir: %s' % base_dir + print 'Using destination dir: %s' % drpm_dir + + rpm_files = getFileList(base_dir, ".rpm", []) + drpm_files = getFileList(base_dir, ".drpm", []) + if not len(rpm_files): print ' Nothing found.' return changed -# print srcfiles[0] -# assert srcfiles[0].startswith(srcdir) - # Check whether dstdir exists, and if it doesn't, create it - if not os.access(dstdir, os.F_OK): - os.makedirs(dstdir, 0755) - elif not os.access(dstdir, os.W_OK): - print 'ERROR: Unable to write to %s' % dstdir + # Check whether rel_dir exists, and if it doesn't, create it + if not os.access(drpm_dir, os.F_OK): + os.makedirs(drpm_dir, 0755) + elif not os.access(drpm_dir, os.W_OK): + print 'ERROR: Unable to write to %s' % drpm_dir sys.exit(1) - # Create map: rpm %name -> list of tuples (filename,name,e,v,r) - newestsrcrpms = {} - for f in srcfiles: - hdr = rpmUtils.miscutils.hdrFromPackage(ts, f) - nm = hdr['name'] + "." + hdr['arch'] - n = hdr['name'] - a = hdr['arch'] - v = hdr['version'] - r = hdr['release'] - e = hdr['epoch'] - if e is None: - e = 0 - newestsrcrpms.setdefault(nm,[]) - newestsrcrpms[nm].append((f,n,e,v,r,a)) - - # Now purge old src.rpm unless their %name matches a white-list pattern. - for l in newestsrcrpms.itervalues(): - x = len(l) - - if x > 1: - def sortByEVR(fnevr1, fnevr2): - (f1,n1,e1,v1,r1,a1) = fnevr1 - (f2,n2,e2,v2,r2,a2) = fnevr2 - rc = rpmUtils.miscutils.compareEVR((e1,v1,r1),(e2,v2,r2)) - if rc == 0: - return 0 - if rc > 0: - return -1 - if rc < 0: - return 1 + # Add all rpms to PackageList + rpm_list = PackageList(base_dir, drpm_dir, DRPMWORTHKEEPINGTHRESH, DEBUG) + for f in rpm_files: + try: + hdr = rpmUtils.miscutils.hdrFromPackage(ts, f) + except: + print "Unable to open %s" % f + else: + nm = hdr['name'] + "." + hdr['arch'] + e = hdr['epoch'] + if e is None: + e = "0" + + nevra = (hdr['name'], e, hdr['version'], hdr['release'], hdr['arch']) + rpm = RpmItem(nevra, f) + rpm_list.addRpm(rpm) + + # Add all deltarpms to PackageList + for f in drpm_files: + try: + hdr = rpmUtils.miscutils.hdrFromPackage(ts, f) + except: + print "Unable to open %s" % f + else: + (sn, se, sv, sr) = getDeltaNevr(f) - l.sort(sortByEVR) # highest first in list - - # Generate delta rpm - finished_rpms = [] - if len(l) > 2: - genDeltaRPM(ts, l[0], l[1], srcdir, dstdir) - for rpm in l[2:]: - genDeltaRPM(ts, l[0], rpm, srcdir, dstdir, l[1]) - else: - for rpm in l[1:]: - genDeltaRPM(ts, l[0], rpm, srcdir, dstdir) - + nm = hdr['name'] + "." + hdr['arch'] + e = hdr['epoch'] + if e is None: + e = "0" + + dst_nevra = (hdr['name'], e, hdr['version'], hdr['release'], hdr['arch']) + src_nevra = (sn, se, sv, sr, hdr['arch']) + drpm = DrpmItem(src_nevra, dst_nevra, f) + rpm_list.addDrpm(drpm) + + if DEBUG: + rpm_list.dump() + + # Build deltarpms + rpm_list.makeDeltaRpms() + def main(bin_rpm_path, delta_rpm_path): assert rpmUtils.miscutils.compareEVR((1,2,3),(1,2,0)) > 0 assert rpmUtils.miscutils.compareEVR((0,1,2),(0,1,2)) == 0 @@ -207,14 +184,6 @@ if __name__ == '__main__': bin_rpm_path = sys.argv[1] delta_rpm_path = sys.argv[2] - #### cfg = Utils.load_config_module(sys.argv[1]) - - #### Utils.signer_gid_check(cfg.signersgid) - #### os.umask(cfg.signersumask) - - #### for dist in sys.argv[2:]: - #### if not cfg.archdict.has_key(dist): - #### print "No distribution release named '%s' found" % dist - #### sys.exit(errno.EINVAL) main(bin_rpm_path, delta_rpm_path) sys.exit(0) + |