From bdfb4037667c6af7d0537f9bc36b33d55d86e059 Mon Sep 17 00:00:00 2001 From: Jonathan Dieter Date: Tue, 3 Apr 2007 16:26:09 +0300 Subject: Add Ahmed's patch to rebuild rpms in separate thread. Complete rebuild of createprestorepo Signed-off-by: Jonathan Dieter --- createprestorepo/createprestorepo.py | 308 ----------------------------------- 1 file changed, 308 deletions(-) delete mode 100755 createprestorepo/createprestorepo.py (limited to 'createprestorepo/createprestorepo.py') diff --git a/createprestorepo/createprestorepo.py b/createprestorepo/createprestorepo.py deleted file mode 100755 index a217045..0000000 --- a/createprestorepo/createprestorepo.py +++ /dev/null @@ -1,308 +0,0 @@ -#!/usr/bin/python -t -# -*- mode: Python; indent-tabs-mode: nil; -*- -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -import errno, os, sys, gzip -import fnmatch, re -import rpmUtils.transaction, rpmUtils.miscutils -import commands, libxml2 -import dumpMetadata -from dumpMetadata import _gzipOpen, getChecksum -#### import Utils - -DEBUG = True -#### Utils.setdebug(DEBUG) - -SUFFIX='drpm' -DRPMWORTHKEEPINGTHRESH=0.5 -DEBUG=0 -REPODATA="repodata" -REPOFILE="presto.xml" -REPOMDFILE="prestomd.xml" -SUM_TYPE="sha" - -def XML_start_newrpm(node, (f, n, e, v, r, a), srcdir_len): - newrpm_node = node.newChild(None, "package", None) - newrpm_node.newProp("type", "rpm") - newrpm_node.newChild(None, "name", n) - newrpm_node.newChild(None, "arch", str(a)) - version = newrpm_node.newChild(None, "version", None) - version.newProp("epoch", str(e)) - version.newProp("ver", str(v)) - version.newProp("rel", str(r)) - deltas = newrpm_node.newChild(None, "deltas", None) - return deltas - -def XML_oldrpm(newrpm_node, drpm_file, oldrpm, newrpm, sequence, size): - (f, n, e, v, r, a) = oldrpm - (nf, nn, ne, nv, nr, na) = newrpm - oldrpm_node = newrpm_node.newChild(None, "oldrpm", None) - checksum = getChecksum(SUM_TYPE, drpm_file) - if n != nn: - oldrpm_node.newChild(None, "name", n) - if a != na: - oldrpm_node.newChild(None, "arch", str(a)) - version = oldrpm_node.newChild(None, "version", None) - if e != ne: - version.newProp("epoch", str(e)) - if v != nv: - version.newProp("ver", str(v)) - version.newProp("rel", str(r)) - oldrpm_node.newChild(None, "drpm_filename", drpm_file) - oldrpm_node.newChild(None, "size", str(size)) - oldrpm_node.newChild(None, "sequence", str(sequence)) - cs_node = oldrpm_node.newChild(None, "checksum", str(checksum)) - cs_node.newProp("type", SUM_TYPE) - -def startXML(): - basedoc = libxml2.newDoc("1.0") - baseroot = basedoc.newChild(None, "metadata", None) - basens = baseroot.newNs('http://linux.duke.edu/metadata/common', None) - formatns = baseroot.newNs('http://linux.duke.edu/metadata/rpm', 'rpm') - baseroot.setNs(basens) - return (basedoc, baseroot) - -def endXML(xmldoc, filename, srcdir, compressed=True): - if compressed: - outfile = _gzipOpen("%s%s/%s.gz" % (srcdir, REPODATA, filename), "w") - output = xmldoc.serialize('UTF-8', 1) - outfile.write(output) - outfile.close() - else: - xmldoc.saveFormatFileEnc("%s%s/%s" % (srcdir, REPODATA, filename), 'UTF-8', 1) - xmldoc.freeDoc() - -def repoXML(srcdir): - """generate the repomd.xml file that stores the info on the other files""" - repodoc = libxml2.newDoc("1.0") - reporoot = repodoc.newChild(None, "repomd", None) - repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None) - reporoot.setNs(repons) - repofilepath = "%s%s/%s" % (srcdir, REPODATA, REPOMDFILE) - filename = "%s%s/%s.gz" % (srcdir, REPODATA, REPOFILE) - filetype = "deltas" - zfo = _gzipOpen(filename, "rb") - uncsum = getChecksum(SUM_TYPE, zfo) - zfo.close() - csum = getChecksum(SUM_TYPE, filename) - timestamp = os.stat(filename)[8] - data = reporoot.newChild(None, 'data', None) - data.newProp('type', filetype) - location = data.newChild(None, 'location', None) - location.newProp('href', "%s/%s.gz" % (REPODATA, REPOFILE)) - checksum = data.newChild(None, 'checksum', csum) - checksum.newProp('type', SUM_TYPE) - timestamp = data.newChild(None, 'timestamp', str(timestamp)) - unchecksum = data.newChild(None, 'open-checksum', uncsum) - unchecksum.newProp('type', SUM_TYPE) - endXML(repodoc, REPOMDFILE, srcdir, False) - -def genDeltaRPM(ts, newrpm, oldrpm, is_new_package, srcdir, dstdir, locroot): - (f1,n1,e1,v1,r1,a1) = newrpm - (f2,n2,e2,v2,r2,a2) = oldrpm - hdr = rpmUtils.miscutils.hdrFromPackage(ts,f1) - arch = hdr['arch'] - v12 = "_".join([v1,v2]) - r12 = "_".join([r1,r2]) - deltaRPMName= '%s/%s.%s.%s' % (dstdir, "-".join([n1,v12,r12]), a1, SUFFIX) - if DEBUG: - print "DEBUG " + deltaCommand - # If the drpm doesn't exists, make it, else skip it - if os.path.exists("%s%s" % (srcdir, deltaRPMName)): - dsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName)) - if e1 == e2: - print 'Using pre-generated delta rpm for %s.%s - %s.%s => %s.%s' % (n1, a1, v2, r2, v1, r1) - else: - print 'Using pre-generated delta rpm for %s.%s - %s:%s.%s => %s:%s.%s' % (n1, a1, e2, v2, r2, e1, v1, r1) - # Get checksum - seqfile = open("%s%s.seq" % (srcdir, deltaRPMName), "r") - sequence = seqfile.read()[:-1] - sequence = sequence[sequence.rfind("-")+1:] - seqfile.close() - if is_new_package: - locroot = XML_start_newrpm(locroot, newrpm, len(srcdir)) - is_new_package = False - XML_oldrpm(locroot, deltaRPMName, oldrpm, newrpm, sequence, dsize) - if DEBUG: - print "DEBUG skipping %s" % (deltaRPMName) - elif os.path.exists("%s%s.dontdelta" % (srcdir, deltaRPMName)): - pass - else: - deltaCommand = 'makedeltarpm -s %s%s.seq %s %s %s%s' % (srcdir, deltaRPMName, f2, f1, srcdir, deltaRPMName) - (code, out) = commands.getstatusoutput(deltaCommand) - if code: - #raise Exception("genDeltaRPM: exitcode was %s - Reported Error: %s" % (code, out)) - print "Error genDeltaRPM for %s: exitcode was %s - Reported Error: %s" % (n1, code, out) - - # Get size - dsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName)) - - # Get checksum - seqfile = open("%s%s.seq" % (srcdir, deltaRPMName), "r") - sequence = seqfile.read()[:-1] - sequence = sequence[sequence.rfind("-")+1:] - seqfile.close() - - # Check whether or not we should keep the drpm - if not drpmIsWorthKeeping(deltaRPMName, f1, srcdir): - if DEBUG: - print 'deleting %s' % (deltaRPMName) - try: - os.unlink("%s%s" % (srcdir, deltaRPMName)) - except Exception, e: - print "Error deleting deltarpm %s" % (deltaRPMName), str(e) - try: - os.unlink("%s%s.seq" % (srcdir, deltaRPMName)) - except Exception, e: - print "Error deleting checksum %s.seq" % (deltaRPMName), str(e) - f = open("%s%s.dontdelta" % (srcdir, deltaRPMName), "w") - f.close() - else: - if e1 == e2: - print 'Generated delta rpm for %s.%s - %s.%s => %s.%s' % (n1, a1, v2, r2, v1, r1) - else: - print 'Generated delta rpm for %s.%s - %s:%s.%s => %s:%s.%s' % (n1, a1, e2, v2, r2, e1, v1, r1) - - if is_new_package: - locroot = XML_start_newrpm(locroot, newrpm, len(srcdir)) - is_new_package = False - XML_oldrpm(locroot, deltaRPMName, oldrpm, newrpm, sequence, dsize) - return (is_new_package, locroot) - -def drpmIsWorthKeeping(deltaRPMName, newrpm, srcdir): - newsize = os.path.getsize(newrpm) - drpmsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName)) - # Delete the drpm if it's too large - if drpmsize > DRPMWORTHKEEPINGTHRESH * newsize: - return False - return True - -def createPrestoRepo(srcdir, dstdir): - ts = rpmUtils.transaction.initReadOnlyTransaction() - changed = False - - # Create list of .rpm files. - # We don't use "glob", so sub-directories are supported. - print 'Using source dir: %s' % srcdir - print 'Using destination dir: %s' % dstdir - if dstdir[-1] == "/": - dstdir = dstdir[:-1] - srcfiles = [] - for root, dirs, files in os.walk(srcdir): - for f in fnmatch.filter(files,'*.rpm'): - srcfiles.append(os.path.join(root,f)) - if not len(srcfiles): - print ' Nothing found.' - return changed - assert srcfiles[0].startswith(srcdir) - - # Check whether dstdir exists, and if it doesn't, create it - if not os.access(dstdir, os.F_OK): - os.makedirs(dstdir, 0755) - elif not os.access(dstdir, os.W_OK): - print 'ERROR: Unable to write to %s' % dstdir - sys.exit(1) - - # Check whether REPODATA exists, and if it doesn't, create it - if not os.access("%s%s" % (srcdir, REPODATA), os.F_OK): - os.makedirs("%s%s" % (srcdir, REPODATA), 0755) - elif not os.access(dstdir, os.W_OK): - print 'ERROR: Unable to write to %s' % REPODATA - sys.exit(1) - - # Create XML document -# xmldoc = libxml2.newDoc("1.0") -# xmlroot = xmldoc - (xmldoc, xmlroot) = startXML() - - # Create map: rpm %name -> list of tuples (filename,name,e,v,r) - newestsrcrpms = {} - for f in srcfiles: - hdr = rpmUtils.miscutils.hdrFromPackage(ts, f) - nm = hdr['name'] + "." + hdr['arch'] - n = hdr['name'] - a = hdr['arch'] - v = hdr['version'] - r = hdr['release'] - e = hdr['epoch'] - if e is None: - e = 0 - newestsrcrpms.setdefault(nm,[]) - newestsrcrpms[nm].append((f,n,e,v,r,a)) - - # Now purge old src.rpm unless their %name matches a white-list pattern. - for l in newestsrcrpms.itervalues(): - x = len(l) - - if x > 1: - def sortByEVR(fnevr1, fnevr2): - (f1,n1,e1,v1,r1,a1) = fnevr1 - (f2,n2,e2,v2,r2,a2) = fnevr2 - rc = rpmUtils.miscutils.compareEVR((e1,v1,r1),(e2,v2,r2)) - if rc == 0: - return 0 - if rc > 0: - return -1 - if rc < 0: - return 1 - - l.sort(sortByEVR) # highest first in list - - # Generate delta rpm - is_new_package = True - locroot = xmlroot - for rpm in l[1:]: - (is_new_package, locroot) = genDeltaRPM(ts, l[0], rpm, is_new_package, srcdir, dstdir, locroot) - - if not len(srcfiles): - print 'WARNING: No .rpms left. Stopping here.' - return changed - - # Write out end of deltas.xml file - endXML(xmldoc, REPOFILE, srcdir, True) - repoXML(srcdir) - - # Examine binary repository directories and remove everything which - # is missing its corresponding src.rpm. - return changed - - -def main(bin_rpm_path, delta_rpm_path): - assert rpmUtils.miscutils.compareEVR((1,2,3),(1,2,0)) > 0 - assert rpmUtils.miscutils.compareEVR((0,1,2),(0,1,2)) == 0 - assert rpmUtils.miscutils.compareEVR((1,2,3),(4,0,99)) < 0 - - return createPrestoRepo(bin_rpm_path, delta_rpm_path) - - -if __name__ == '__main__': - if len(sys.argv) < 2: - print 'Usage: %s \n' % os.path.basename(sys.argv[0]) - sys.exit(errno.EINVAL) - bin_rpm_path = sys.argv[1] - delta_rpm_path = sys.argv[2] - - #### cfg = Utils.load_config_module(sys.argv[1]) - - #### Utils.signer_gid_check(cfg.signersgid) - #### os.umask(cfg.signersumask) - - #### for dist in sys.argv[2:]: - #### if not cfg.archdict.has_key(dist): - #### print "No distribution release named '%s' found" % dist - #### sys.exit(errno.EINVAL) - main(bin_rpm_path, delta_rpm_path) - sys.exit(0) -- cgit