summaryrefslogtreecommitdiffstats
path: root/createprestorepo/gendeltarpms.py
diff options
context:
space:
mode:
authorJonathan Dieter <jdieter@gmail.com>2007-05-07 21:19:56 +0300
committerJonathan Dieter <jdieter@gmail.com>2007-05-07 21:19:56 +0300
commitd60f28d66709c0c7fed26d19b1b7fb2c55439507 (patch)
tree338bdb4ed55b2290dd9577f2987d882a605c2505 /createprestorepo/gendeltarpms.py
parentc69bc9af8f3459610e810970c6222aee643e5594 (diff)
downloadpresto-d60f28d66709c0c7fed26d19b1b7fb2c55439507.zip
presto-d60f28d66709c0c7fed26d19b1b7fb2c55439507.tar.gz
presto-d60f28d66709c0c7fed26d19b1b7fb2c55439507.tar.xz
First release of createprestorepo, prunerepo and prunedrpms
Signed-off-by: Jonathan Dieter <jdieter@gmail.com>
Diffstat (limited to 'createprestorepo/gendeltarpms.py')
-rwxr-xr-xcreateprestorepo/gendeltarpms.py275
1 files changed, 122 insertions, 153 deletions
diff --git a/createprestorepo/gendeltarpms.py b/createprestorepo/gendeltarpms.py
index 5e0226f..fc4a9ca 100755
--- a/createprestorepo/gendeltarpms.py
+++ b/createprestorepo/gendeltarpms.py
@@ -20,17 +20,77 @@ import fnmatch, re
import rpmUtils.transaction, rpmUtils.miscutils
import commands, libxml2
import string
+from dumpMetadata import byteranges
+from packagelist import RpmItem, DrpmItem, PackageList
-SUFFIX='drpm'
-DRPMWORTHKEEPINGTHRESH=0.5
-DEBUG=1
+import gzip
+from zlib import error as zlibError
+from gzip import write32u, FNAME
-def getFileList(basepath, path, ext, filelist):
+
+DRPMWORTHKEEPINGTHRESH=0.7
+DEBUG=0
+
+def getDeltaNevr(filename):
+ """Get nevr for src rpm of a deltarpm. Return a tuple of (n, e, v, r)"""
+ def _getLength(in_data):
+ length = 0
+ for val in in_data:
+ length = length * 256
+ length += ord(val)
+ return length
+
+ def _stringToVersion(strng):
+ i = strng.find(':')
+ if i != -1:
+ epoch = strng[:i]
+ else:
+ epoch = '0'
+ j = strng.find('-')
+ if j != -1:
+ if strng[i + 1:j] == '':
+ version = None
+ else:
+ version = strng[i + 1:j]
+ release = strng[j + 1:]
+ else:
+ if strng[i + 1:] == '':
+ version = None
+ else:
+ version = strng[i + 1:]
+ release = None
+ return (epoch, version, release)
+
+ def _stringToNEVR(string):
+ i = string.rfind("-", 0, string.rfind("-")-1)
+ name = string[:i]
+ (epoch, ver, rel) = _stringToVersion(string[i+1:])
+ return (name, epoch, ver, rel)
+
+ (range_start, range_end) = byteranges(filename)
+ fd = open(filename, "r")
+ fd.seek(range_end)
+ try:
+ compobj = gzip.GzipFile("", "rb", 9, fd)
+ except:
+ raise zlibError("Data not stored in gzip format")
+
+ if compobj.read(4)[:3] != "DLT":
+ raise Exception("Not a deltarpm")
+
+ nevr_length = _getLength(compobj.read(4))
+ nevr = compobj.read(nevr_length).strip("\x00")
+ oldnevr = _stringToNEVR(nevr)
+ compobj.close()
+ return oldnevr
+
+
+def getFileList(path, ext, filelist):
"""Return all files in path matching ext, store them in filelist,
recurse dirs. Returns a list object"""
extlen = len(ext)
- totalpath = os.path.normpath(os.path.join(basepath, path))
+ totalpath = os.path.normpath(path)
try:
dir_list = os.listdir(totalpath)
except OSError, e:
@@ -39,159 +99,76 @@ def getFileList(basepath, path, ext, filelist):
for d in dir_list:
if os.path.isdir(totalpath + '/' + d):
- filelist = getFileList(basepath, os.path.join(path, d), ext, filelist)
- else:
- if string.lower(d[-extlen:]) == '%s' % (ext):
- if totalpath.find(basepath) == 0:
- if basepath != "":
- relativepath = totalpath.replace(basepath, "", 1)
- else:
- relativepath = totalpath
- relativepath = relativepath.lstrip("/")
- filelist.append(os.path.join(relativepath, d))
- else:
- raise "basepath '%s' not found in path '%s'" % (basepath, totalpath)
+ filelist = getFileList(os.path.join(totalpath, d), ext, filelist)
+ elif string.lower(d[-extlen:]) == '%s' % (ext):
+ filelist.append(os.path.join(totalpath, d))
return filelist
-
-def genDeltaRPM(ts, newrpm, oldrpm, srcdir, dstdir, old_delta = None):
- (f1,n1,e1,v1,r1,a1) = newrpm
- (f2,n2,e2,v2,r2,a2) = oldrpm
- hdr = rpmUtils.miscutils.hdrFromPackage(ts,f1)
- arch = hdr['arch']
- v12 = "_".join([v1,v2])
- r12 = "_".join([r1,r2])
- deltaRPMName= '%s/%s.%s.%s' % (dstdir, "-".join([n1,v12,r12]), a1, SUFFIX)
-
- # If the drpm doesn't exist, make it, else skip it
- if os.path.exists("%s%s" % (srcdir, deltaRPMName)) or os.path.exists("%s%s.dontdelta" % (srcdir, deltaRPMName)):
-# if e1 == e2:
-# print 'Using pre-generated delta rpm for %s.%s - %s.%s => %s.%s' % (n1, a1, v2, r2, v1, r1)
-# else:
-# print 'Using pre-generated delta rpm for %s.%s - %s:%s.%s => %s:%s.%s' % (n1, a1, e2, v2, r2, e1, v1, r1)
- if DEBUG:
- print "DEBUG skipping %s" % (deltaRPMName)
- else:
- generated = False
- prep_made = False
- if old_delta != None:
- (df,dn,de,dv,dr,da) = old_delta
- v1d = "_".join([v1,dv])
- r1d = "_".join([r1,dr])
- src_deltaRPMFile= '%s%s/%s.%s.%s' % (srcdir, dstdir, "-".join([n1,v1d,r1d]), a1, SUFFIX)
- vd2 = "_".join([dv,v2])
- rd2 = "_".join([dr,r2])
- old_deltaRPMFile= '%s%s/%s.%s.%s' % (srcdir, dstdir, "-".join([n1,vd2,rd2]), a1, SUFFIX)
- if os.access(old_deltaRPMFile, os.R_OK) and os.access(src_deltaRPMFile, os.R_OK):
- deltaCommand = 'combinedeltarpm %s %s %s%s' % (old_deltaRPMFile, src_deltaRPMFile, srcdir, deltaRPMName)
- if DEBUG:
- print deltaCommand
- prep_made = True
-
- if not prep_made:
- deltaCommand = 'makedeltarpm %s %s %s%s' % (f2, f1, srcdir, deltaRPMName)
- if DEBUG:
- print deltaCommand
-
- (code, out) = commands.getstatusoutput(deltaCommand)
- if code:
- #raise Exception("genDeltaRPM: exitcode was %s - Reported Error: %s" % (code, out))
- print "Error genDeltaRPM for %s: exitcode was %s - Reported Error: %s" % (n1, code, out)
- else:
- # Check whether or not we should keep the drpm
- if not drpmIsWorthKeeping(deltaRPMName, f1, srcdir):
- if DEBUG:
- print 'deleting %s' % (deltaRPMName)
- try:
- os.unlink("%s%s" % (srcdir, deltaRPMName))
- except Exception, e:
- print "Error deleting deltarpm %s" % (deltaRPMName), str(e)
- f = open("%s%s.dontdelta" % (srcdir, deltaRPMName), "w")
- f.close()
- else:
- if e1 == e2:
- print 'Generated delta rpm for %s.%s - %s-%s => %s-%s' % (n1, a1, v2, r2, v1, r1)
- else:
- print 'Generated delta rpm for %s.%s - %s:%s-%s => %s:%s-%s' % (n1, a1, e2, v2, r2, e1, v1, r1)
- return deltaRPMName
-
-def drpmIsWorthKeeping(deltaRPMName, newrpm, srcdir):
- newsize = os.path.getsize(newrpm)
- drpmsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName))
- # Delete the drpm if it's too large
- if drpmsize > DRPMWORTHKEEPINGTHRESH * newsize:
- return False
- return True
-
-def createPrestoRepo(srcdir, dstdir):
+def createPrestoRepo(base_dir, drpm_dir):
ts = rpmUtils.transaction.initReadOnlyTransaction()
changed = False
# Create list of .rpm files.
# We don't use "glob", so sub-directories are supported.
- print 'Using source dir: %s' % srcdir
- print 'Using destination dir: %s' % dstdir
- if dstdir[-1] == "/":
- dstdir = dstdir[:-1]
- srcfiles = []
- srcfiles = getFileList("", srcdir, ".rpm", [])
- if not len(srcfiles):
+ print 'Using base dir: %s' % base_dir
+ print 'Using destination dir: %s' % drpm_dir
+
+ rpm_files = getFileList(base_dir, ".rpm", [])
+ drpm_files = getFileList(base_dir, ".drpm", [])
+ if not len(rpm_files):
print ' Nothing found.'
return changed
-# print srcfiles[0]
-# assert srcfiles[0].startswith(srcdir)
- # Check whether dstdir exists, and if it doesn't, create it
- if not os.access(dstdir, os.F_OK):
- os.makedirs(dstdir, 0755)
- elif not os.access(dstdir, os.W_OK):
- print 'ERROR: Unable to write to %s' % dstdir
+ # Check whether rel_dir exists, and if it doesn't, create it
+ if not os.access(drpm_dir, os.F_OK):
+ os.makedirs(drpm_dir, 0755)
+ elif not os.access(drpm_dir, os.W_OK):
+ print 'ERROR: Unable to write to %s' % drpm_dir
sys.exit(1)
- # Create map: rpm %name -> list of tuples (filename,name,e,v,r)
- newestsrcrpms = {}
- for f in srcfiles:
- hdr = rpmUtils.miscutils.hdrFromPackage(ts, f)
- nm = hdr['name'] + "." + hdr['arch']
- n = hdr['name']
- a = hdr['arch']
- v = hdr['version']
- r = hdr['release']
- e = hdr['epoch']
- if e is None:
- e = 0
- newestsrcrpms.setdefault(nm,[])
- newestsrcrpms[nm].append((f,n,e,v,r,a))
-
- # Now purge old src.rpm unless their %name matches a white-list pattern.
- for l in newestsrcrpms.itervalues():
- x = len(l)
-
- if x > 1:
- def sortByEVR(fnevr1, fnevr2):
- (f1,n1,e1,v1,r1,a1) = fnevr1
- (f2,n2,e2,v2,r2,a2) = fnevr2
- rc = rpmUtils.miscutils.compareEVR((e1,v1,r1),(e2,v2,r2))
- if rc == 0:
- return 0
- if rc > 0:
- return -1
- if rc < 0:
- return 1
+ # Add all rpms to PackageList
+ rpm_list = PackageList(base_dir, drpm_dir, DRPMWORTHKEEPINGTHRESH, DEBUG)
+ for f in rpm_files:
+ try:
+ hdr = rpmUtils.miscutils.hdrFromPackage(ts, f)
+ except:
+ print "Unable to open %s" % f
+ else:
+ nm = hdr['name'] + "." + hdr['arch']
+ e = hdr['epoch']
+ if e is None:
+ e = "0"
+
+ nevra = (hdr['name'], e, hdr['version'], hdr['release'], hdr['arch'])
+ rpm = RpmItem(nevra, f)
+ rpm_list.addRpm(rpm)
+
+ # Add all deltarpms to PackageList
+ for f in drpm_files:
+ try:
+ hdr = rpmUtils.miscutils.hdrFromPackage(ts, f)
+ except:
+ print "Unable to open %s" % f
+ else:
+ (sn, se, sv, sr) = getDeltaNevr(f)
- l.sort(sortByEVR) # highest first in list
-
- # Generate delta rpm
- finished_rpms = []
- if len(l) > 2:
- genDeltaRPM(ts, l[0], l[1], srcdir, dstdir)
- for rpm in l[2:]:
- genDeltaRPM(ts, l[0], rpm, srcdir, dstdir, l[1])
- else:
- for rpm in l[1:]:
- genDeltaRPM(ts, l[0], rpm, srcdir, dstdir)
-
+ nm = hdr['name'] + "." + hdr['arch']
+ e = hdr['epoch']
+ if e is None:
+ e = "0"
+
+ dst_nevra = (hdr['name'], e, hdr['version'], hdr['release'], hdr['arch'])
+ src_nevra = (sn, se, sv, sr, hdr['arch'])
+ drpm = DrpmItem(src_nevra, dst_nevra, f)
+ rpm_list.addDrpm(drpm)
+
+ if DEBUG:
+ rpm_list.dump()
+
+ # Build deltarpms
+ rpm_list.makeDeltaRpms()
+
def main(bin_rpm_path, delta_rpm_path):
assert rpmUtils.miscutils.compareEVR((1,2,3),(1,2,0)) > 0
assert rpmUtils.miscutils.compareEVR((0,1,2),(0,1,2)) == 0
@@ -207,14 +184,6 @@ if __name__ == '__main__':
bin_rpm_path = sys.argv[1]
delta_rpm_path = sys.argv[2]
- #### cfg = Utils.load_config_module(sys.argv[1])
-
- #### Utils.signer_gid_check(cfg.signersgid)
- #### os.umask(cfg.signersumask)
-
- #### for dist in sys.argv[2:]:
- #### if not cfg.archdict.has_key(dist):
- #### print "No distribution release named '%s' found" % dist
- #### sys.exit(errno.EINVAL)
main(bin_rpm_path, delta_rpm_path)
sys.exit(0)
+