Add Ahmed's patch to rebuild rpms in separate thread.

Complete rebuild of createprestorepo

Signed-off-by: Jonathan Dieter <jdieter@gmail.com>

1 files changed, 0 insertions, 308 deletions

diff --git a/createprestorepo/createprestorepo.py b/createprestorepo/createprestorepo.py
deleted file mode 100755
index a217045..0000000
--- a/createprestorepo/createprestorepo.py
+++ /dev/null
@@ -1,308 +0,0 @@
-#!/usr/bin/python -t
-# -*- mode: Python; indent-tabs-mode: nil; -*-
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; either version 2 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
-
-import errno, os, sys, gzip
-import fnmatch, re
-import rpmUtils.transaction, rpmUtils.miscutils
-import commands, libxml2
-import dumpMetadata
-from dumpMetadata import _gzipOpen, getChecksum
-#### import Utils
-
-DEBUG = True
-#### Utils.setdebug(DEBUG)
-
-SUFFIX='drpm'
-DRPMWORTHKEEPINGTHRESH=0.5
-DEBUG=0
-REPODATA="repodata"
-REPOFILE="presto.xml"
-REPOMDFILE="prestomd.xml"
-SUM_TYPE="sha"
-
-def XML_start_newrpm(node, (f, n, e, v, r, a), srcdir_len):
-    newrpm_node = node.newChild(None, "package", None)
-    newrpm_node.newProp("type", "rpm")
-    newrpm_node.newChild(None, "name", n)
-    newrpm_node.newChild(None, "arch", str(a))
-    version = newrpm_node.newChild(None, "version", None)
-    version.newProp("epoch", str(e))
-    version.newProp("ver", str(v))
-    version.newProp("rel", str(r))
-    deltas = newrpm_node.newChild(None, "deltas", None)
-    return deltas
-
-def XML_oldrpm(newrpm_node, drpm_file, oldrpm, newrpm, sequence, size):
-    (f, n, e, v, r, a) = oldrpm
-    (nf, nn, ne, nv, nr, na) = newrpm
-    oldrpm_node = newrpm_node.newChild(None, "oldrpm", None)
-    checksum = getChecksum(SUM_TYPE, drpm_file)
-    if n != nn:
-        oldrpm_node.newChild(None, "name", n)
-    if a != na:
-        oldrpm_node.newChild(None, "arch", str(a))
-    version = oldrpm_node.newChild(None, "version", None)
-    if e != ne:
-        version.newProp("epoch", str(e))
-    if v != nv:
-        version.newProp("ver", str(v))
-    version.newProp("rel", str(r))
-    oldrpm_node.newChild(None, "drpm_filename", drpm_file)
-    oldrpm_node.newChild(None, "size", str(size))
-    oldrpm_node.newChild(None, "sequence", str(sequence))
-    cs_node = oldrpm_node.newChild(None, "checksum", str(checksum))
-    cs_node.newProp("type", SUM_TYPE)
-
-def startXML():
-    basedoc = libxml2.newDoc("1.0")
-    baseroot =  basedoc.newChild(None, "metadata", None)
-    basens = baseroot.newNs('http://linux.duke.edu/metadata/common', None)
-    formatns = baseroot.newNs('http://linux.duke.edu/metadata/rpm', 'rpm')
-    baseroot.setNs(basens)
-    return (basedoc, baseroot)
-
-def endXML(xmldoc, filename, srcdir, compressed=True):
-    if compressed:
-        outfile = _gzipOpen("%s%s/%s.gz" % (srcdir, REPODATA, filename), "w")
-        output = xmldoc.serialize('UTF-8', 1)
-        outfile.write(output)
-        outfile.close()
-    else:
-        xmldoc.saveFormatFileEnc("%s%s/%s" % (srcdir, REPODATA, filename), 'UTF-8', 1)
-    xmldoc.freeDoc()
-
-def repoXML(srcdir):
-    """generate the repomd.xml file that stores the info on the other files"""
-    repodoc = libxml2.newDoc("1.0")
-    reporoot = repodoc.newChild(None, "repomd", None)
-    repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None)
-    reporoot.setNs(repons)
-    repofilepath = "%s%s/%s" % (srcdir, REPODATA, REPOMDFILE)
-    filename = "%s%s/%s.gz" % (srcdir, REPODATA, REPOFILE)
-    filetype = "deltas"
-    zfo = _gzipOpen(filename, "rb")
-    uncsum = getChecksum(SUM_TYPE, zfo)
-    zfo.close()
-    csum = getChecksum(SUM_TYPE, filename)
-    timestamp = os.stat(filename)[8]
-    data = reporoot.newChild(None, 'data', None)
-    data.newProp('type', filetype)
-    location = data.newChild(None, 'location', None)
-    location.newProp('href', "%s/%s.gz" % (REPODATA, REPOFILE))
-    checksum = data.newChild(None, 'checksum', csum)
-    checksum.newProp('type', SUM_TYPE)
-    timestamp = data.newChild(None, 'timestamp', str(timestamp))
-    unchecksum = data.newChild(None, 'open-checksum', uncsum)
-    unchecksum.newProp('type', SUM_TYPE)
-    endXML(repodoc, REPOMDFILE, srcdir, False)
-     
-def genDeltaRPM(ts, newrpm, oldrpm, is_new_package, srcdir, dstdir, locroot):
-    (f1,n1,e1,v1,r1,a1) = newrpm
-    (f2,n2,e2,v2,r2,a2) = oldrpm
-    hdr = rpmUtils.miscutils.hdrFromPackage(ts,f1)
-    arch = hdr['arch']
-    v12 = "_".join([v1,v2])
-    r12 = "_".join([r1,r2])
-    deltaRPMName= '%s/%s.%s.%s' % (dstdir, "-".join([n1,v12,r12]), a1, SUFFIX)
-    if DEBUG:
-        print "DEBUG " + deltaCommand
-    # If the drpm doesn't exists, make it, else skip it
-    if os.path.exists("%s%s" % (srcdir, deltaRPMName)):
-        dsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName))
-        if e1 == e2:
-            print 'Using pre-generated delta rpm for %s.%s - %s.%s => %s.%s' % (n1, a1, v2, r2, v1, r1)
-        else:
-            print 'Using pre-generated delta rpm for %s.%s - %s:%s.%s => %s:%s.%s' % (n1, a1, e2, v2, r2, e1, v1, r1)
-        # Get checksum
-        seqfile = open("%s%s.seq" % (srcdir, deltaRPMName), "r")
-        sequence = seqfile.read()[:-1]
-        sequence = sequence[sequence.rfind("-")+1:]
-        seqfile.close()
-        if is_new_package:
-            locroot = XML_start_newrpm(locroot, newrpm, len(srcdir))
-            is_new_package = False
-        XML_oldrpm(locroot, deltaRPMName, oldrpm, newrpm, sequence, dsize)
-        if DEBUG:
-            print "DEBUG skipping %s" % (deltaRPMName)
-    elif os.path.exists("%s%s.dontdelta" % (srcdir, deltaRPMName)):
-        pass
-    else:
-        deltaCommand = 'makedeltarpm -s %s%s.seq %s %s %s%s' % (srcdir, deltaRPMName, f2, f1, srcdir, deltaRPMName)
-        (code, out) = commands.getstatusoutput(deltaCommand)
-        if code:
-            #raise Exception("genDeltaRPM: exitcode was %s - Reported Error: %s" % (code, out))
-            print "Error genDeltaRPM for %s: exitcode was %s - Reported Error: %s" % (n1, code, out)
-            
-        # Get size
-        dsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName))
-
-        # Get checksum
-        seqfile = open("%s%s.seq" % (srcdir, deltaRPMName), "r")
-        sequence = seqfile.read()[:-1]
-        sequence = sequence[sequence.rfind("-")+1:]
-        seqfile.close()
-            
-        # Check whether or not we should keep the drpm
-        if not drpmIsWorthKeeping(deltaRPMName, f1, srcdir):
-            if DEBUG:
-                print 'deleting %s' % (deltaRPMName)
-            try:
-                os.unlink("%s%s" % (srcdir, deltaRPMName))
-            except Exception, e:
-                print "Error deleting deltarpm %s" % (deltaRPMName), str(e)
-            try:
-                os.unlink("%s%s.seq" % (srcdir, deltaRPMName))
-            except Exception, e:
-                print "Error deleting checksum %s.seq" % (deltaRPMName), str(e)
-            f = open("%s%s.dontdelta" % (srcdir, deltaRPMName), "w")
-            f.close()
-        else:
-            if e1 == e2:
-                print 'Generated delta rpm for %s.%s - %s.%s => %s.%s' % (n1, a1, v2, r2, v1, r1)
-            else:
-                print 'Generated delta rpm for %s.%s - %s:%s.%s => %s:%s.%s' % (n1, a1, e2, v2, r2, e1, v1, r1)
-
-            if is_new_package:
-                locroot = XML_start_newrpm(locroot, newrpm, len(srcdir))
-                is_new_package = False
-            XML_oldrpm(locroot, deltaRPMName, oldrpm, newrpm, sequence, dsize)
-    return (is_new_package, locroot)
-
-def drpmIsWorthKeeping(deltaRPMName, newrpm, srcdir):
-    newsize = os.path.getsize(newrpm)
-    drpmsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName))
-    # Delete the drpm if it's too large
-    if drpmsize > DRPMWORTHKEEPINGTHRESH * newsize:
-        return False
-    return True
-
-def createPrestoRepo(srcdir, dstdir):
-    ts = rpmUtils.transaction.initReadOnlyTransaction()
-    changed = False
-    
-    # Create list of .rpm files.
-    # We don't use "glob", so sub-directories are supported.
-    print 'Using source dir: %s' % srcdir
-    print 'Using destination dir: %s' % dstdir
-    if dstdir[-1] == "/":
-        dstdir = dstdir[:-1]
-    srcfiles = []
-    for root, dirs, files in os.walk(srcdir):
-        for f in fnmatch.filter(files,'*.rpm'):
-            srcfiles.append(os.path.join(root,f))
-    if not len(srcfiles):
-        print '  Nothing found.'
-        return changed
-    assert srcfiles[0].startswith(srcdir)
-
-    # Check whether dstdir exists, and if it doesn't, create it
-    if not os.access(dstdir, os.F_OK):
-        os.makedirs(dstdir, 0755)
-    elif not os.access(dstdir, os.W_OK):
-        print 'ERROR: Unable to write to %s' % dstdir
-        sys.exit(1)
-    
-    # Check whether REPODATA exists, and if it doesn't, create it
-    if not os.access("%s%s" % (srcdir, REPODATA), os.F_OK):
-        os.makedirs("%s%s" % (srcdir, REPODATA), 0755)
-    elif not os.access(dstdir, os.W_OK):
-        print 'ERROR: Unable to write to %s' % REPODATA
-        sys.exit(1)
-
-    # Create XML document
-#    xmldoc = libxml2.newDoc("1.0")
-#    xmlroot = xmldoc
-    (xmldoc, xmlroot) = startXML()
-
-    # Create map: rpm %name -> list of tuples (filename,name,e,v,r)
-    newestsrcrpms = {}
-    for f in srcfiles:
-        hdr = rpmUtils.miscutils.hdrFromPackage(ts, f)
-        nm = hdr['name'] + "." + hdr['arch']
-        n = hdr['name']
-        a = hdr['arch']
-        v = hdr['version']
-        r = hdr['release']
-        e = hdr['epoch']
-        if e is None:
-            e = 0
-        newestsrcrpms.setdefault(nm,[])
-        newestsrcrpms[nm].append((f,n,e,v,r,a))
-    
-    # Now purge old src.rpm unless their %name matches a white-list pattern.
-    for l in newestsrcrpms.itervalues():
-        x = len(l)
-
-        if x > 1:
-            def sortByEVR(fnevr1, fnevr2):
-                (f1,n1,e1,v1,r1,a1) = fnevr1
-                (f2,n2,e2,v2,r2,a2) = fnevr2
-                rc = rpmUtils.miscutils.compareEVR((e1,v1,r1),(e2,v2,r2))
-                if rc == 0:
-                    return 0
-                if rc > 0:
-                    return -1
-                if rc < 0:
-                    return 1
-            
-            l.sort(sortByEVR)  # highest first in list
-          
-            # Generate delta rpm
-            is_new_package = True
-            locroot = xmlroot
-            for rpm in l[1:]:	
-                (is_new_package, locroot) = genDeltaRPM(ts, l[0], rpm, is_new_package, srcdir, dstdir, locroot)
-
-    if not len(srcfiles):
-        print 'WARNING: No .rpms left. Stopping here.'
-        return changed
-
-    # Write out end of deltas.xml file
-    endXML(xmldoc, REPOFILE, srcdir, True)
-    repoXML(srcdir)
-    
-    # Examine binary repository directories and remove everything which
-    # is missing its corresponding src.rpm.
-    return changed
-
-
-def main(bin_rpm_path, delta_rpm_path):
-    assert rpmUtils.miscutils.compareEVR((1,2,3),(1,2,0)) > 0
-    assert rpmUtils.miscutils.compareEVR((0,1,2),(0,1,2)) == 0
-    assert rpmUtils.miscutils.compareEVR((1,2,3),(4,0,99)) < 0
-
-    return createPrestoRepo(bin_rpm_path, delta_rpm_path)
-
-
-if __name__ == '__main__':
-    if len(sys.argv) < 2:
-        print 'Usage: %s <bin_rpm_dir> <delta_rpm_dir> \n' % os.path.basename(sys.argv[0])
-        sys.exit(errno.EINVAL)
-    bin_rpm_path = sys.argv[1]
-    delta_rpm_path = sys.argv[2]
-
-    #### cfg = Utils.load_config_module(sys.argv[1])
-
-    #### Utils.signer_gid_check(cfg.signersgid)
-    #### os.umask(cfg.signersumask)
-    
-    #### for dist in sys.argv[2:]:
-    ####     if not cfg.archdict.has_key(dist):
-    ####         print "No distribution release named '%s' found" % dist
-    ####         sys.exit(errno.EINVAL)
-    main(bin_rpm_path, delta_rpm_path)
-    sys.exit(0)