diff options
Diffstat (limited to 'makerepo/createprestorepo.py')
| -rwxr-xr-x | makerepo/createprestorepo.py | 308 |
1 files changed, 308 insertions, 0 deletions
diff --git a/makerepo/createprestorepo.py b/makerepo/createprestorepo.py new file mode 100755 index 0000000..0fee5b4 --- /dev/null +++ b/makerepo/createprestorepo.py @@ -0,0 +1,308 @@ +#!/usr/bin/python -t +# -*- mode: Python; indent-tabs-mode: nil; -*- +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +import errno, os, sys, gzip +import fnmatch, re +import rpmUtils.transaction, rpmUtils.miscutils +import commands, libxml2 +import dumpMetadata +from dumpMetadata import _gzipOpen, getChecksum +#### import Utils + +DEBUG = False +#### Utils.setdebug(DEBUG) + +SUFFIX='drpm' +DRPMWORTHKEEPINGTHRESH=0.5 +DEBUG=0 +REPODATA="repodata" +REPOFILE="presto.xml" +REPOMDFILE="prestomd.xml" +SUM_TYPE="sha" + +def XML_start_newrpm(node, (f, n, e, v, r, a), srcdir_len): + newrpm_node = node.newChild(None, "package", None) + newrpm_node.newProp("type", "rpm") + newrpm_node.newChild(None, "name", n) + newrpm_node.newChild(None, "arch", str(a)) + version = newrpm_node.newChild(None, "version", None) + version.newProp("epoch", str(e)) + version.newProp("ver", str(v)) + version.newProp("rel", str(r)) + deltas = newrpm_node.newChild(None, "deltas", None) + return deltas + +def XML_oldrpm(newrpm_node, drpm_file, oldrpm, newrpm, sequence, size): + (f, n, e, v, r, a) = oldrpm + (nf, nn, ne, nv, nr, na) = newrpm + oldrpm_node = newrpm_node.newChild(None, "oldrpm", None) + checksum = getChecksum(SUM_TYPE, drpm_file) + if n != nn: + oldrpm_node.newChild(None, "name", n) + if a != na: + oldrpm_node.newChild(None, "arch", str(a)) + version = oldrpm_node.newChild(None, "version", None) + if e != ne: + version.newProp("epoch", str(e)) + if v != nv: + version.newProp("ver", str(v)) + version.newProp("rel", str(r)) + oldrpm_node.newChild(None, "drpm_filename", drpm_file) + oldrpm_node.newChild(None, "size", str(size)) + oldrpm_node.newChild(None, "sequence", str(sequence)) + cs_node = oldrpm_node.newChild(None, "checksum", str(checksum)) + cs_node.newProp("type", SUM_TYPE) + +def startXML(): + basedoc = libxml2.newDoc("1.0") + baseroot = basedoc.newChild(None, "metadata", None) + basens = baseroot.newNs('http://linux.duke.edu/metadata/common', None) + formatns = baseroot.newNs('http://linux.duke.edu/metadata/rpm', 'rpm') + baseroot.setNs(basens) + return (basedoc, baseroot) + +def endXML(xmldoc, filename, srcdir, compressed=True): + if compressed: + outfile = _gzipOpen("%s%s/%s.gz" % (srcdir, REPODATA, filename), "w") + output = xmldoc.serialize('UTF-8', 1) + outfile.write(output) + outfile.close() + else: + xmldoc.saveFormatFileEnc("%s%s/%s" % (srcdir, REPODATA, filename), 'UTF-8', 1) + xmldoc.freeDoc() + +def repoXML(srcdir): + """generate the repomd.xml file that stores the info on the other files""" + repodoc = libxml2.newDoc("1.0") + reporoot = repodoc.newChild(None, "repomd", None) + repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None) + reporoot.setNs(repons) + repofilepath = "%s%s/%s" % (srcdir, REPODATA, REPOMDFILE) + filename = "%s%s/%s.gz" % (srcdir, REPODATA, REPOFILE) + filetype = "deltas" + zfo = _gzipOpen(filename, "rb") + uncsum = getChecksum(SUM_TYPE, zfo) + zfo.close() + csum = getChecksum(SUM_TYPE, filename) + timestamp = os.stat(filename)[8] + data = reporoot.newChild(None, 'data', None) + data.newProp('type', filetype) + location = data.newChild(None, 'location', None) + location.newProp('href', "%s/%s.gz" % (REPODATA, REPOFILE)) + checksum = data.newChild(None, 'checksum', csum) + checksum.newProp('type', SUM_TYPE) + timestamp = data.newChild(None, 'timestamp', str(timestamp)) + unchecksum = data.newChild(None, 'open-checksum', uncsum) + unchecksum.newProp('type', SUM_TYPE) + endXML(repodoc, REPOMDFILE, srcdir, False) + +def genDeltaRPM(ts, newrpm, oldrpm, is_new_package, srcdir, dstdir, locroot): + (f1,n1,e1,v1,r1,a1) = newrpm + (f2,n2,e2,v2,r2,a2) = oldrpm + hdr = rpmUtils.miscutils.hdrFromPackage(ts,f1) + arch = hdr['arch'] + v12 = "_".join([v1,v2]) + r12 = "_".join([r1,r2]) + deltaRPMName= '%s/%s.%s.%s' % (dstdir, "-".join([n1,v12,r12]), a1, SUFFIX) + if DEBUG: + print "DEBUG " + deltaCommand + # If the drpm doesn't exists, make it, else skip it + if os.path.exists("%s%s" % (srcdir, deltaRPMName)): + dsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName)) + if e1 == e2: + print 'Using pre-generated delta rpm for %s.%s - %s.%s => %s.%s' % (n1, a1, v2, r2, v1, r1) + else: + print 'Using pre-generated delta rpm for %s.%s - %s:%s.%s => %s:%s.%s' % (n1, a1, e2, v2, r2, e1, v1, r1) + # Get checksum + seqfile = open("%s%s.seq" % (srcdir, deltaRPMName), "r") + sequence = seqfile.read()[:-1] + sequence = sequence[sequence.rfind("-")+1:] + seqfile.close() + if is_new_package: + locroot = XML_start_newrpm(locroot, newrpm, len(srcdir)) + is_new_package = False + XML_oldrpm(locroot, deltaRPMName, oldrpm, newrpm, sequence, dsize) + if DEBUG: + print "DEBUG skipping %s" % (deltaRPMName) + elif os.path.exists("%s%s.dontdelta" % (srcdir, deltaRPMName)) or os.path.getsize(f1) > 70000000: + pass + else: + deltaCommand = 'makedeltarpm -s %s%s.seq %s %s %s%s' % (srcdir, deltaRPMName, f2, f1, srcdir, deltaRPMName) + (code, out) = commands.getstatusoutput(deltaCommand) + if code: + #raise Exception("genDeltaRPM: exitcode was %s - Reported Error: %s" % (code, out)) + print "Error genDeltaRPM for %s: exitcode was %s - Reported Error: %s" % (n1, code, out) + + # Get size + dsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName)) + + # Get checksum + seqfile = open("%s%s.seq" % (srcdir, deltaRPMName), "r") + sequence = seqfile.read()[:-1] + sequence = sequence[sequence.rfind("-")+1:] + seqfile.close() + + # Check whether or not we should keep the drpm + if not drpmIsWorthKeeping(deltaRPMName, f1, srcdir): + if DEBUG: + print 'deleting %s' % (deltaRPMName) + try: + os.unlink("%s%s" % (srcdir, deltaRPMName)) + except Exception, e: + print "Error deleting deltarpm %s" % (deltaRPMName), str(e) + try: + os.unlink("%s%s.seq" % (srcdir, deltaRPMName)) + except Exception, e: + print "Error deleting checksum %s.seq" % (deltaRPMName), str(e) + f = open("%s%s.dontdelta" % (srcdir, deltaRPMName), "w") + f.close() + else: + if e1 == e2: + print 'Generated delta rpm for %s.%s - %s.%s => %s.%s' % (n1, a1, v2, r2, v1, r1) + else: + print 'Generated delta rpm for %s.%s - %s:%s.%s => %s:%s.%s' % (n1, a1, e2, v2, r2, e1, v1, r1) + + if is_new_package: + locroot = XML_start_newrpm(locroot, newrpm, len(srcdir)) + is_new_package = False + XML_oldrpm(locroot, deltaRPMName, oldrpm, newrpm, sequence, dsize) + return (is_new_package, locroot) + +def drpmIsWorthKeeping(deltaRPMName, newrpm, srcdir): + newsize = os.path.getsize(newrpm) + drpmsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName)) + # Delete the drpm if it's too large + if drpmsize > DRPMWORTHKEEPINGTHRESH * newsize: + return False + return True + +def createPrestoRepo(srcdir, dstdir): + ts = rpmUtils.transaction.initReadOnlyTransaction() + changed = False + + # Create list of .rpm files. + # We don't use "glob", so sub-directories are supported. + print 'Using source dir: %s' % srcdir + print 'Using destination dir: %s' % dstdir + if dstdir[-1] == "/": + dstdir = dstdir[:-1] + srcfiles = [] + for root, dirs, files in os.walk(srcdir): + for f in fnmatch.filter(files,'*.rpm'): + srcfiles.append(os.path.join(root,f)) + if not len(srcfiles): + print ' Nothing found.' + return changed + assert srcfiles[0].startswith(srcdir) + + # Check whether dstdir exists, and if it doesn't, create it + if not os.access(dstdir, os.F_OK): + os.makedirs(dstdir, 0755) + elif not os.access(dstdir, os.W_OK): + print 'ERROR: Unable to write to %s' % dstdir + sys.exit(1) + + # Check whether REPODATA exists, and if it doesn't, create it + if not os.access("%s%s" % (srcdir, REPODATA), os.F_OK): + os.makedirs("%s%s" % (srcdir, REPODATA), 0755) + elif not os.access(dstdir, os.W_OK): + print 'ERROR: Unable to write to %s' % REPODATA + sys.exit(1) + + # Create XML document +# xmldoc = libxml2.newDoc("1.0") +# xmlroot = xmldoc + (xmldoc, xmlroot) = startXML() + + # Create map: rpm %name -> list of tuples (filename,name,e,v,r) + newestsrcrpms = {} + for f in srcfiles: + hdr = rpmUtils.miscutils.hdrFromPackage(ts, f) + nm = hdr['name'] + "." + hdr['arch'] + n = hdr['name'] + a = hdr['arch'] + v = hdr['version'] + r = hdr['release'] + e = hdr['epoch'] + if e is None: + e = 0 + newestsrcrpms.setdefault(nm,[]) + newestsrcrpms[nm].append((f,n,e,v,r,a)) + + # Now purge old src.rpm unless their %name matches a white-list pattern. + for l in newestsrcrpms.itervalues(): + x = len(l) + + if x > 1: + def sortByEVR(fnevr1, fnevr2): + (f1,n1,e1,v1,r1,a1) = fnevr1 + (f2,n2,e2,v2,r2,a2) = fnevr2 + rc = rpmUtils.miscutils.compareEVR((e1,v1,r1),(e2,v2,r2)) + if rc == 0: + return 0 + if rc > 0: + return -1 + if rc < 0: + return 1 + + l.sort(sortByEVR) # highest first in list + + # Generate delta rpm + is_new_package = True + locroot = xmlroot + for rpm in l[1:]: + (is_new_package, locroot) = genDeltaRPM(ts, l[0], rpm, is_new_package, srcdir, dstdir, locroot) + + if not len(srcfiles): + print 'WARNING: No .rpms left. Stopping here.' + return changed + + # Write out end of deltas.xml file + endXML(xmldoc, REPOFILE, srcdir, True) + repoXML(srcdir) + + # Examine binary repository directories and remove everything which + # is missing its corresponding src.rpm. + return changed + + +def main(bin_rpm_path, delta_rpm_path): + assert rpmUtils.miscutils.compareEVR((1,2,3),(1,2,0)) > 0 + assert rpmUtils.miscutils.compareEVR((0,1,2),(0,1,2)) == 0 + assert rpmUtils.miscutils.compareEVR((1,2,3),(4,0,99)) < 0 + + return createPrestoRepo(bin_rpm_path, delta_rpm_path) + + +if __name__ == '__main__': + if len(sys.argv) < 2: + print 'Usage: %s <bin_rpm_dir> <delta_rpm_dir> \n' % os.path.basename(sys.argv[0]) + sys.exit(errno.EINVAL) + bin_rpm_path = sys.argv[1] + delta_rpm_path = sys.argv[2] + + #### cfg = Utils.load_config_module(sys.argv[1]) + + #### Utils.signer_gid_check(cfg.signersgid) + #### os.umask(cfg.signersumask) + + #### for dist in sys.argv[2:]: + #### if not cfg.archdict.has_key(dist): + #### print "No distribution release named '%s' found" % dist + #### sys.exit(errno.EINVAL) + main(bin_rpm_path, delta_rpm_path) + sys.exit(0) |