path: root/makerepo/createprestorepo.py

#!/usr/bin/python -t
# -*- mode: Python; indent-tabs-mode: nil; -*-
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA

import errno, os, sys, gzip
import fnmatch, re
import rpmUtils.transaction, rpmUtils.miscutils
import commands, libxml2
import dumpMetadata
from dumpMetadata import _gzipOpen, getChecksum
#### import Utils

DEBUG = False
#### Utils.setdebug(DEBUG)

SUFFIX='drpm'
DRPMWORTHKEEPINGTHRESH=0.5
DEBUG=0
REPODATA="repodata"
REPOFILE="presto.xml"
REPOMDFILE="prestomd.xml"
SUM_TYPE="sha"

def XML_start_newrpm(node, (f, n, e, v, r, a), srcdir_len):
    newrpm_node = node.newChild(None, "package", None)
    newrpm_node.newProp("type", "rpm")
    newrpm_node.newChild(None, "name", n)
    newrpm_node.newChild(None, "arch", str(a))
    version = newrpm_node.newChild(None, "version", None)
    version.newProp("epoch", str(e))
    version.newProp("ver", str(v))
    version.newProp("rel", str(r))
    deltas = newrpm_node.newChild(None, "deltas", None)
    return deltas

def XML_oldrpm(newrpm_node, drpm_file, oldrpm, newrpm, sequence, size):
    (f, n, e, v, r, a) = oldrpm
    (nf, nn, ne, nv, nr, na) = newrpm
    oldrpm_node = newrpm_node.newChild(None, "oldrpm", None)
    checksum = getChecksum(SUM_TYPE, drpm_file)
    if n != nn:
        oldrpm_node.newChild(None, "name", n)
    if a != na:
        oldrpm_node.newChild(None, "arch", str(a))
    version = oldrpm_node.newChild(None, "version", None)
    if e != ne:
        version.newProp("epoch", str(e))
    if v != nv:
        version.newProp("ver", str(v))
    version.newProp("rel", str(r))
    oldrpm_node.newChild(None, "drpm_filename", drpm_file)
    oldrpm_node.newChild(None, "size", str(size))
    oldrpm_node.newChild(None, "sequence", str(sequence))
    cs_node = oldrpm_node.newChild(None, "checksum", str(checksum))
    cs_node.newProp("type", SUM_TYPE)

def startXML():
    basedoc = libxml2.newDoc("1.0")
    baseroot =  basedoc.newChild(None, "metadata", None)
    basens = baseroot.newNs('http://linux.duke.edu/metadata/common', None)
    formatns = baseroot.newNs('http://linux.duke.edu/metadata/rpm', 'rpm')
    baseroot.setNs(basens)
    return (basedoc, baseroot)

def endXML(xmldoc, filename, srcdir, compressed=True):
    if compressed:
        outfile = _gzipOpen("%s%s/%s.gz" % (srcdir, REPODATA, filename), "w")
        output = xmldoc.serialize('UTF-8', 1)
        outfile.write(output)
        outfile.close()
    else:
        xmldoc.saveFormatFileEnc("%s%s/%s" % (srcdir, REPODATA, filename), 'UTF-8', 1)
    xmldoc.freeDoc()

def repoXML(srcdir):
    """generate the repomd.xml file that stores the info on the other files"""
    repodoc = libxml2.newDoc("1.0")
    reporoot = repodoc.newChild(None, "repomd", None)
    repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None)
    reporoot.setNs(repons)
    repofilepath = "%s%s/%s" % (srcdir, REPODATA, REPOMDFILE)
    filename = "%s%s/%s.gz" % (srcdir, REPODATA, REPOFILE)
    filetype = "deltas"
    zfo = _gzipOpen(filename, "rb")
    uncsum = getChecksum(SUM_TYPE, zfo)
    zfo.close()
    csum = getChecksum(SUM_TYPE, filename)
    timestamp = os.stat(filename)[8]
    data = reporoot.newChild(None, 'data', None)
    data.newProp('type', filetype)
    location = data.newChild(None, 'location', None)
    location.newProp('href', "%s/%s.gz" % (REPODATA, REPOFILE))
    checksum = data.newChild(None, 'checksum', csum)
    checksum.newProp('type', SUM_TYPE)
    timestamp = data.newChild(None, 'timestamp', str(timestamp))
    unchecksum = data.newChild(None, 'open-checksum', uncsum)
    unchecksum.newProp('type', SUM_TYPE)
    endXML(repodoc, REPOMDFILE, srcdir, False)
     
def genDeltaRPM(ts, newrpm, oldrpm, is_new_package, srcdir, dstdir, locroot):
    (f1,n1,e1,v1,r1,a1) = newrpm
    (f2,n2,e2,v2,r2,a2) = oldrpm
    hdr = rpmUtils.miscutils.hdrFromPackage(ts,f1)
    arch = hdr['arch']
    v12 = "_".join([v1,v2])
    r12 = "_".join([r1,r2])
    deltaRPMName= '%s/%s.%s.%s' % (dstdir, "-".join([n1,v12,r12]), a1, SUFFIX)
    if DEBUG:
        print "DEBUG " + deltaCommand
    # If the drpm doesn't exists, make it, else skip it
    if os.path.exists("%s%s" % (srcdir, deltaRPMName)):
        dsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName))
        if e1 == e2:
            print 'Using pre-generated delta rpm for %s.%s - %s.%s => %s.%s' % (n1, a1, v2, r2, v1, r1)
        else:
            print 'Using pre-generated delta rpm for %s.%s - %s:%s.%s => %s:%s.%s' % (n1, a1, e2, v2, r2, e1, v1, r1)
        # Get checksum
        seqfile = open("%s%s.seq" % (srcdir, deltaRPMName), "r")
        sequence = seqfile.read()[:-1]
        sequence = sequence[sequence.rfind("-")+1:]
        seqfile.close()
        if is_new_package:
            locroot = XML_start_newrpm(locroot, newrpm, len(srcdir))
            is_new_package = False
        XML_oldrpm(locroot, deltaRPMName, oldrpm, newrpm, sequence, dsize)
        if DEBUG:
            print "DEBUG skipping %s" % (deltaRPMName)
    elif os.path.exists("%s%s.dontdelta" % (srcdir, deltaRPMName)) or os.path.getsize(f1) > 70000000:
        pass
    else:
        deltaCommand = 'makedeltarpm -s %s%s.seq %s %s %s%s' % (srcdir, deltaRPMName, f2, f1, srcdir, deltaRPMName)
        (code, out) = commands.getstatusoutput(deltaCommand)
        if code:
            #raise Exception("genDeltaRPM: exitcode was %s - Reported Error: %s" % (code, out))
            print "Error genDeltaRPM for %s: exitcode was %s - Reported Error: %s" % (n1, code, out)
            
        # Get size
        dsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName))

        # Get checksum
        seqfile = open("%s%s.seq" % (srcdir, deltaRPMName), "r")
        sequence = seqfile.read()[:-1]
        sequence = sequence[sequence.rfind("-")+1:]
        seqfile.close()
            
        # Check whether or not we should keep the drpm
        if not drpmIsWorthKeeping(deltaRPMName, f1, srcdir):
            if DEBUG:
                print 'deleting %s' % (deltaRPMName)
            try:
                os.unlink("%s%s" % (srcdir, deltaRPMName))
            except Exception, e:
                print "Error deleting deltarpm %s" % (deltaRPMName), str(e)
            try:
                os.unlink("%s%s.seq" % (srcdir, deltaRPMName))
            except Exception, e:
                print "Error deleting checksum %s.seq" % (deltaRPMName), str(e)
            f = open("%s%s.dontdelta" % (srcdir, deltaRPMName), "w")
            f.close()
        else:
            if e1 == e2:
                print 'Generated delta rpm for %s.%s - %s.%s => %s.%s' % (n1, a1, v2, r2, v1, r1)
            else:
                print 'Generated delta rpm for %s.%s - %s:%s.%s => %s:%s.%s' % (n1, a1, e2, v2, r2, e1, v1, r1)

            if is_new_package:
                locroot = XML_start_newrpm(locroot, newrpm, len(srcdir))
                is_new_package = False
            XML_oldrpm(locroot, deltaRPMName, oldrpm, newrpm, sequence, dsize)
    return (is_new_package, locroot)

def drpmIsWorthKeeping(deltaRPMName, newrpm, srcdir):
    newsize = os.path.getsize(newrpm)
    drpmsize = os.path.getsize("%s%s" % (srcdir, deltaRPMName))
    # Delete the drpm if it's too large
    if drpmsize > DRPMWORTHKEEPINGTHRESH * newsize:
        return False
    return True

def createPrestoRepo(srcdir, dstdir):
    ts = rpmUtils.transaction.initReadOnlyTransaction()
    changed = False
    
    # Create list of .rpm files.
    # We don't use "glob", so sub-directories are supported.
    print 'Using source dir: %s' % srcdir
    print 'Using destination dir: %s' % dstdir
    if dstdir[-1] == "/":
        dstdir = dstdir[:-1]
    srcfiles = []
    for root, dirs, files in os.walk(srcdir):
        for f in fnmatch.filter(files,'*.rpm'):
            srcfiles.append(os.path.join(root,f))
    if not len(srcfiles):
        print '  Nothing found.'
        return changed
    assert srcfiles[0].startswith(srcdir)
    
    # Check whether dstdir exists, and if it doesn't, create it
    if not os.access(dstdir, os.F_OK):
        os.makedirs(dstdir, 0755)
    elif not os.access(dstdir, os.W_OK):
        print 'ERROR: Unable to write to %s' % dstdir
        sys.exit(1)
    
    # Check whether REPODATA exists, and if it doesn't, create it
    if not os.access("%s%s" % (srcdir, REPODATA), os.F_OK):
        os.makedirs("%s%s" % (srcdir, REPODATA), 0755)
    elif not os.access(dstdir, os.W_OK):
        print 'ERROR: Unable to write to %s' % REPODATA
        sys.exit(1)

    # Create XML document
#    xmldoc = libxml2.newDoc("1.0")
#    xmlroot = xmldoc
    (xmldoc, xmlroot) = startXML()

    # Create map: rpm %name -> list of tuples (filename,name,e,v,r)
    newestsrcrpms = {}
    for f in srcfiles:
        hdr = rpmUtils.miscutils.hdrFromPackage(ts, f)
        nm = hdr['name'] + "." + hdr['arch']
        n = hdr['name']
        a = hdr['arch']
        v = hdr['version']
        r = hdr['release']
        e = hdr['epoch']
        if e is None:
            e = 0
        newestsrcrpms.setdefault(nm,[])
        newestsrcrpms[nm].append((f,n,e,v,r,a))
    
    # Now purge old src.rpm unless their %name matches a white-list pattern.
    for l in newestsrcrpms.itervalues():
        x = len(l)

        if x > 1:
            def sortByEVR(fnevr1, fnevr2):
                (f1,n1,e1,v1,r1,a1) = fnevr1
                (f2,n2,e2,v2,r2,a2) = fnevr2
                rc = rpmUtils.miscutils.compareEVR((e1,v1,r1),(e2,v2,r2))
                if rc == 0:
                    return 0
                if rc > 0:
                    return -1
                if rc < 0:
                    return 1
            
            l.sort(sortByEVR)  # highest first in list
          
            # Generate delta rpm
            is_new_package = True
            locroot = xmlroot
            for rpm in l[1:]:
                (is_new_package, locroot) = genDeltaRPM(ts, l[0], rpm, is_new_package, srcdir, dstdir, locroot)

    if not len(srcfiles):
        print 'WARNING: No .rpms left. Stopping here.'
        return changed

    # Write out end of deltas.xml file
    endXML(xmldoc, REPOFILE, srcdir, True)
    repoXML(srcdir)
    
    # Examine binary repository directories and remove everything which
    # is missing its corresponding src.rpm.
    return changed


def main(bin_rpm_path, delta_rpm_path):
    assert rpmUtils.miscutils.compareEVR((1,2,3),(1,2,0)) > 0
    assert rpmUtils.miscutils.compareEVR((0,1,2),(0,1,2)) == 0
    assert rpmUtils.miscutils.compareEVR((1,2,3),(4,0,99)) < 0

    return createPrestoRepo(bin_rpm_path, delta_rpm_path)


if __name__ == '__main__':
    if len(sys.argv) < 2:
        print 'Usage: %s <bin_rpm_dir> <delta_rpm_dir> \n' % os.path.basename(sys.argv[0])
        sys.exit(errno.EINVAL)
    bin_rpm_path = sys.argv[1]
    delta_rpm_path = sys.argv[2]

    #### cfg = Utils.load_config_module(sys.argv[1])

    #### Utils.signer_gid_check(cfg.signersgid)
    #### os.umask(cfg.signersumask)
    
    #### for dist in sys.argv[2:]:
    ####     if not cfg.archdict.has_key(dist):
    ####         print "No distribution release named '%s' found" % dist
    ####         sys.exit(errno.EINVAL)
    main(bin_rpm_path, delta_rpm_path)
    sys.exit(0)