From 31f0565879cfc4aeeccf1f86f32b76c8aa34aa20 Mon Sep 17 00:00:00 2001 From: Will Woods Date: Thu, 19 Feb 2009 19:35:09 -0500 Subject: debuginfofs-mirror now does downloading, unpacking, and linking. --- debuginfofs-mirror | 177 ++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 142 insertions(+), 35 deletions(-) diff --git a/debuginfofs-mirror b/debuginfofs-mirror index 5433a77..5370fbe 100755 --- a/debuginfofs-mirror +++ b/debuginfofs-mirror @@ -20,9 +20,12 @@ import os import sys +import shutil import logging import optparse import ConfigParser +# Why, yes, I do arrange imports by line length +from subprocess import Popen, PIPE sys.path.insert(0,'/usr/share/yum-cli') from utils import YumUtilBase from yum.parser import varReplace @@ -40,10 +43,12 @@ except (IOError, sys.exit(1) # FIXME get these from commandline (and validate 'em) +savecache = False conf['releasever']='10' conf['basearch']='i386' -repolist=('fedora','fedora-updates','fedora-updates-testing') +#repolist=('fedora','fedora-updates','fedora-updates-testing') +repolist=('fedora-updates-testing',) class DebuginfoFSDownloader(YumUtilBase): NAME = 'debuginfofs-mirror' @@ -56,46 +61,148 @@ class DebuginfoFSDownloader(YumUtilBase): DebuginfoFSDownloader.USAGE) self.logger = logging.getLogger("yum.verbose.cli.debuginfofs") -y = DebuginfoFSDownloader() -y.doConfigSetup() -# TODO check for RepoError -# Kinda stupid, setting up and then immediately tearing down all repos.. -y.repos.disableRepo('*') -# No yum cache needed, since we're saving the packages ourselves -y.repos.setCacheDir(conf['cachedir']) -y.conf.cache = 0 -y.repos.setCache(0) -# TODO: optionally allow caching? -# kind of a cheap hack -my_yumvar = y.conf.yumvar -my_yumvar.update(conf) +def yum_setup(): + y = DebuginfoFSDownloader() + y.doConfigSetup() + # TODO check for RepoError + # Kinda stupid, setting up and then immediately tearing down all repos.. + y.repos.disableRepo('*') + # No yum cache needed, since we're saving the packages ourselves + y.repos.setCacheDir(conf['cachedir']) + y.conf.cache = 0 + y.repos.setCache(0) + # TODO: optionally allow caching? + # FIXME repo cache data is ending up in $CWD + return y + +def fix_perms(targetdir): + '''Make all files readable, and all directories read+execute''' + for top, dirs, files in os.walk(targetdir): + for d in dirs: + i = os.path.join(top, d) + mode = os.stat(i)[0] + os.chmod(i, mode | 0555) + for f in files: + i = os.path.join(top, f) + if not os.path.islink(i): + mode = os.stat(i)[0] + os.chmod(i, mode | 0444) + +def unpack_rpm(rpm, targetdir): + created_dir = False + if not os.path.isdir(targetdir): + os.makedirs(targetdir) + created_dir = True -for repoid in repolist: try: - d = dict([(k,varReplace(v,my_yumvar)) for k,v in c.items(repoid)]) - except ConfigParser.NoSectionError: - print "No section named %s in config" % repoid - continue - if 'mirrorlist' in d: - print "Adding repo %s with mirrorlist %s" % (repoid,d['mirrorlist']) - repo = YumRepository(repoid + '-debuginfofs') - repo.name = repo.id - repo.mirrorlist = d['mirrorlist'] - repo.cache = 0 - repo.enable() - y.repos.add(repo) - y.repos.doSetup(thisrepo=repo.id) + # FIXME: unpack into a tmpdir and then move into place after finishing + # or else interrupted unpacks will leave us with incomplete output + os.chdir(targetdir) + # rpm2cpio $rpm | cpio --quiet -iumd + p1 = Popen(['rpm2cpio',rpm], stdout=PIPE) + p2 = Popen(['cpio','--quiet','-iumd'], stdin=p1.stdout, stdout=PIPE) + output = p2.communicate()[0] # should be empty + if p2.returncode != 0: + raise OSError, "cpio failed: %s output:\n%s" % (str(p2.returncode),output) + # Fix perms so all files are readable + fix_perms(targetdir) + except (IOError, OSError), e: + print str(e) + if created_dir: + print "removing %s" % targetdir + shutil.rmtree(targetdir) + return False + return True + +def mkdebuginfolinks(sourcedir, targetdir): + '''hardlink debuginfo from sourcedir into targetdir''' + count = 0 + for top, dirs, files in os.walk(sourcedir, topdown=True): + if '/usr/lib/debug/.build-id/' not in top: + continue + for u in [os.path.join(top,f) for f in files if f.endswith('.debug')]: + target = os.path.realpath(u) + linkname = u.split('/usr/lib/debug/.build-id/')[1] + newlink = os.path.join(targetdir,linkname) + try: + os.makedirs(os.path.dirname(newlink)) + except OSError, e: + if e.errno != 17: + raise e + if os.path.exists(newlink): + os.unlink(newlink) + os.link(target,newlink) + count += 1 + return count + +# Check permissions +perms_ok = True +for d in (conf['exportdir'], conf['cachedir']): + if not os.access(d,os.W_OK): + perms_ok = False + print "ERROR: you don't have write permission on %s" % d +if not perms_ok: + sys.exit(1) +if __name__ == '__main__': + y = yum_setup() + # kind of a cheap hack + my_yumvar = y.conf.yumvar + my_yumvar.update(conf) + + # Uses y, conf, c, repolist + for repoid in repolist: + try: + d = dict([(k,varReplace(v,my_yumvar)) for k,v in c.items(repoid)]) + except ConfigParser.NoSectionError: + print "No section named %s in config" % repoid + continue + if 'mirrorlist' in d: + print "Adding repo %s with mirrorlist %s" % (repoid,d['mirrorlist']) + repo = YumRepository(repoid + '-debuginfofs') + repo.name = repo.id + repo.mirrorlist = d['mirrorlist'] + repo.cache = 0 + repo.enable() + y.repos.add(repo) + y.repos.doSetup(thisrepo=repo.id) + + print "Downloading to cachedir %s" % conf['cachedir'] + print "Unpacking to exportdir %s" % conf['exportdir'] packages = y.pkgSack.returnPackages() + for p in packages: repo = y.repos.getRepo(p.repoid) remote = p.returnSimple('relativepath') local = os.path.join(conf['cachedir'],os.path.basename(remote)) p.localpath = local - # TODO: Check to see if we've already got it in cache - # TODO: Check to see if it's already unpacked - # Download - #path = repo.getPackage(p) - # TODO: Unpack - # TODO: Remove cached package - # TODO: Hardlink + (n,a,e,v,r) = p.pkgtup + nevra = "%s-%s:%s-%s.%s" % (n,e,v,r,a) + #/var/www/debuginfofs/packages/c/coreutils/coreutils-0:6.12-18.fc10.i386 + newdir = os.path.join(n[0],n,nevra) + targetdir = os.path.join(conf['exportdir'],'packages',newdir) + # Check to see if it's already unpacked + if os.path.isdir(targetdir): + print "Already unpacked %s" % nevra + # TODO optionally rmtree and redownload/unpack + # FIXME: continue, or just skip fetch/unpack and do links anyway? + continue + + # Check to see if we've already got it in cache + if (os.path.exists(local) and + os.path.getsize(local) == int(p.returnSimple('packagesize'))): + print "Already downloaded %s" % nevra + else: + # Download package + print "Downloading %s" % nevra + # FIXME: uninterruptible! + path = repo.getPackage(p) + + print "Unpacking to %s" % newdir + unpack_rpm(local, targetdir) + # Remove cached package now that we've unpacked it + if not savecache: + os.unlink(local) + # Make hardlinks + r=mkdebuginfolinks(targetdir,os.path.join(conf['exportdir'],'build-id')) + print "Linked %i debuginfo file%s" % (r, r != 1 and "s" or "") -- cgit