createprestorepo - full source

Signed-off-by: Jonathan Dieter <jdieter@gmail.com>

3 files changed, 225 insertions, 1 deletions

diff --git a/createprestorepo/Makefile b/createprestorepo/Makefile
index c950686..a7f6004 100644
--- a/createprestorepo/Makefile
+++ b/createprestorepo/Makefile
@@ -2,7 +2,9 @@ clean:
 	rm -f *.pyc *.pyo *~
 
 install:
-	install -m 755 createprestorepo $(DESTDIR)/usr/bin/createprestorepo
+	install -m 755 createprestorepo $(DESTDIR)/usr/bin/
+	install -m 755 makedeltarepo $(DESTDIR)/usr/bin/
 	mkdir -p $(DESTDIR)/usr/share/createprestorepo
 	install -m 755 genprestometadata.py $(DESTDIR)/usr/share/createprestorepo
 	install	-m 755 dumpMetadata.py $(DESTDIR)/usr/share/createprestorepo
+	install -m 755 gendeltarpms.py $(DESTDIR)/usr/share/createprestorepo
diff --git a/createprestorepo/gendeltarpms.py b/createprestorepo/gendeltarpms.py
new file mode 100755
index 0000000..5e0226f
--- /dev/null
+++ b/createprestorepo/gendeltarpms.py
@@ -0,0 +1,220 @@
+#!/usr/bin/python -t
+# -*- mode: Python; indent-tabs-mode: nil; -*-
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
+
+import errno, os, sys, gzip
+import fnmatch, re
+import rpmUtils.transaction, rpmUtils.miscutils
+import commands, libxml2
+import string
+
+SUFFIX='drpm'
+DRPMWORTHKEEPINGTHRESH=0.5
+DEBUG=1
+
+def getFileList(basepath, path, ext, filelist):
+    """Return all files in path matching ext, store them in filelist,
+    recurse dirs. Returns a list object"""
+
+    extlen = len(ext)
+    totalpath = os.path.normpath(os.path.join(basepath, path))
+    try:
+        dir_list = os.listdir(totalpath)
+    except OSError, e:
+        errorprint(_('Error accessing directory %s, %s') % (totalpath, e))
+        sys.exit(1)
+
+    for d in dir_list:
+        if os.path.isdir(totalpath + '/' + d):
+            filelist = getFileList(basepath, os.path.join(path, d), ext, filelist)
+        else:
+            if string.lower(d[-extlen:]) == '%s' % (ext):
+                if totalpath.find(basepath) == 0:
+                    if basepath != "":
+                        relativepath = totalpath.replace(basepath, "", 1)
+                    else:
+                        relativepath = totalpath
+                    relativepath = relativepath.lstrip("/")
+                    filelist.append(os.path.join(relativepath, d))
+                else:
+                    raise "basepath '%s' not found in path '%s'" % (basepath, totalpath)
+
+    return filelist
+
+     
+def genDeltaRPM(ts, newrpm, oldrpm, srcdir, dstdir, old_delta = None):
+    (f1,n1,e1,v1,r1,a1) = newrpm
+    (f2,n2,e2,v2,r2,a2) = oldrpm
+    hdr = rpmUtils.miscutils.hdrFromPackage(ts,f1)
+    arch = hdr['arch']
+    v12 = "_".join([v1,v2])
+    r12 = "_".join([r1,r2])
+    deltaRPMName= '%s/%s.%s.%s' % (dstdir, "-".join([n1,v12,r12]), a1, SUFFIX)
+        
+    # If the drpm doesn't exist, make it, else skip it
+    if os.path.exists("%s%s" % (srcdir, deltaRPMName)) or os.path.exists("%s%s.dontdelta" % (srcdir, deltaRPMName)):
+#        if e1 == e2:
+#            print 'Using pre-generated delta rpm for %s.%s - %s.%s => %s.%s' % (n1, a1, v2, r2, v1, r1)
+#        else:
+#            print 'Using pre-generated delta rpm for %s.%s - %s:%s.%s => %s:%s.%s' % (n1, a1, e2, v2, r2, e1, v1, r1)
+        if DEBUG:
+            print "DEBUG skipping %s" % (deltaRPMName)
+    else:
+        generated = False
+        prep_made = False
+        if old_delta != None:
+            (df,dn,de,dv,dr,da) = old_delta
+            v1d = "_".join([v1,dv])
+            r1d = "_".join([r1,dr])
+            src_deltaRPMFile= '%s%s/%s.%s.%s' % (srcdir, dstdir, "-".join([n1,v1d,r1d]), a1, SUFFIX)
+            vd2 = "_".join([dv,v2])
+            rd2 = "_".join([dr,r2])
+            old_deltaRPMFile= '%s%s/%s.%s.%s' % (srcdir, dstdir, "-".join([n1,vd2,rd2]), a1, SUFFIX)
+            if os.access(old_deltaRPMFile, os.R_OK) and os.access(src_deltaRPMFile, os.R_OK):
+                deltaCommand = 'combinedeltarpm %s %s %s%s' % (old_deltaRPMFile, src_deltaRPMFile, srcdir, deltaRPMName)
+                if DEBUG:
+                    print deltaCommand
+                prep_made = True
+
+        if not prep_made:
+            deltaCommand = 'makedeltarpm %s %s %s%s' % (f2, f1, srcdir, deltaRPMName)
+            if DEBUG:
+                print deltaCommand
+            
+        (code, out) = commands.getstatusoutput(deltaCommand)
+        if code:
+            #raise Exception("genDeltaRPM: exitcode was %s - Reported Error: %s" % (code, out))
+            print "Error genDeltaRPM for %s: exitcode was %s - Reported Error: %s" % (n1, code, out)
+        else:                 
+            # Check whether or not we should keep the drpm
+            if not drpmIsWorthKeeping(deltaRPMName, f1, srcdir):
+                if DEBUG:
+                    print 'deleting %s' % (deltaRPMName)
+                try:
+                    os.unlink("%s%s" % (srcdir, deltaRPMName))
+                except Exception, e:
+                    print "Error deleting deltarpm %s" % (deltaRPMName), str(e)
+                f = open("%s%s.dontdelta" % (srcdir, deltaRPMName), "w")
+                f.close()
+            else:
+                if e1 == e2:
+                    print 'Generated delta rpm for %s.%s - %s-%s => %s-%s' % (n1, a1, v2, r2, v1, r1)
+                else:
+                    print 'Generated delta rpm for %s.%s - %s:%s-%s => %s:%s-%s' % (n1, a1, e2, v2, r2, e1, v1, r1)
+    return deltaRPMName
+
+def drpmIsWorthKeeping(deltaRPMName, newrpm, srcdir):
+    newsize = os.path.getsize(newrpm)
+    drpmsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName))
+    # Delete the drpm if it's too large
+    if drpmsize > DRPMWORTHKEEPINGTHRESH * newsize:
+        return False
+    return True
+
+def createPrestoRepo(srcdir, dstdir):
+    ts = rpmUtils.transaction.initReadOnlyTransaction()
+    changed = False
+    
+    # Create list of .rpm files.
+    # We don't use "glob", so sub-directories are supported.
+    print 'Using source dir: %s' % srcdir
+    print 'Using destination dir: %s' % dstdir
+    if dstdir[-1] == "/":
+        dstdir = dstdir[:-1]
+    srcfiles = []
+    srcfiles = getFileList("", srcdir, ".rpm", [])
+    if not len(srcfiles):
+        print '  Nothing found.'
+        return changed
+#    print srcfiles[0]
+#    assert srcfiles[0].startswith(srcdir)
+    
+    # Check whether dstdir exists, and if it doesn't, create it
+    if not os.access(dstdir, os.F_OK):
+        os.makedirs(dstdir, 0755)
+    elif not os.access(dstdir, os.W_OK):
+        print 'ERROR: Unable to write to %s' % dstdir
+        sys.exit(1)
+    
+    # Create map: rpm %name -> list of tuples (filename,name,e,v,r)
+    newestsrcrpms = {}
+    for f in srcfiles:
+        hdr = rpmUtils.miscutils.hdrFromPackage(ts, f)
+        nm = hdr['name'] + "." + hdr['arch']
+        n = hdr['name']
+        a = hdr['arch']
+        v = hdr['version']
+        r = hdr['release']
+        e = hdr['epoch']
+        if e is None:
+            e = 0
+        newestsrcrpms.setdefault(nm,[])
+        newestsrcrpms[nm].append((f,n,e,v,r,a))
+    
+    # Now purge old src.rpm unless their %name matches a white-list pattern.
+    for l in newestsrcrpms.itervalues():
+        x = len(l)
+
+        if x > 1:
+            def sortByEVR(fnevr1, fnevr2):
+                (f1,n1,e1,v1,r1,a1) = fnevr1
+                (f2,n2,e2,v2,r2,a2) = fnevr2
+                rc = rpmUtils.miscutils.compareEVR((e1,v1,r1),(e2,v2,r2))
+                if rc == 0:
+                    return 0
+                if rc > 0:
+                    return -1
+                if rc < 0:
+                    return 1
+            
+            l.sort(sortByEVR)  # highest first in list
+          
+            # Generate delta rpm
+            finished_rpms = []
+            if len(l) > 2:
+                genDeltaRPM(ts, l[0], l[1], srcdir, dstdir)
+                for rpm in l[2:]:
+                    genDeltaRPM(ts, l[0], rpm, srcdir, dstdir, l[1])
+            else:
+                for rpm in l[1:]:
+                    genDeltaRPM(ts, l[0], rpm, srcdir, dstdir)
+
+def main(bin_rpm_path, delta_rpm_path):
+    assert rpmUtils.miscutils.compareEVR((1,2,3),(1,2,0)) > 0
+    assert rpmUtils.miscutils.compareEVR((0,1,2),(0,1,2)) == 0
+    assert rpmUtils.miscutils.compareEVR((1,2,3),(4,0,99)) < 0
+
+    return createPrestoRepo(bin_rpm_path, delta_rpm_path)
+
+
+if __name__ == '__main__':
+    if len(sys.argv) < 2:
+        print 'Usage: %s <bin_rpm_dir> <delta_rpm_dir> \n' % os.path.basename(sys.argv[0])
+        sys.exit(errno.EINVAL)
+    bin_rpm_path = sys.argv[1]
+    delta_rpm_path = sys.argv[2]
+
+    #### cfg = Utils.load_config_module(sys.argv[1])
+
+    #### Utils.signer_gid_check(cfg.signersgid)
+    #### os.umask(cfg.signersumask)
+    
+    #### for dist in sys.argv[2:]:
+    ####     if not cfg.archdict.has_key(dist):
+    ####         print "No distribution release named '%s' found" % dist
+    ####         sys.exit(errno.EINVAL)
+    main(bin_rpm_path, delta_rpm_path)
+    sys.exit(0)
diff --git a/createprestorepo/makedeltarepo b/createprestorepo/makedeltarepo
new file mode 100755
index 0000000..99bedcc
--- /dev/null
+++ b/createprestorepo/makedeltarepo
@@ -0,0 +1,2 @@
+#!/bin/sh
+exec /usr/share/createprestorepo/gendeltarpms.py "$@"