debuginfofs-mirror now does downloading, unpacking, and linking.

1 files changed, 142 insertions, 35 deletions

diff --git a/debuginfofs-mirror b/debuginfofs-mirror
index 5433a77..5370fbe 100755
--- a/debuginfofs-mirror
+++ b/debuginfofs-mirror
@@ -20,9 +20,12 @@
 
 import os
 import sys
+import shutil
 import logging
 import optparse
 import ConfigParser
+# Why, yes, I do arrange imports by line length
+from subprocess import Popen, PIPE
 sys.path.insert(0,'/usr/share/yum-cli')
 from utils import YumUtilBase
 from yum.parser import varReplace
@@ -40,10 +43,12 @@ except (IOError,
     sys.exit(1)
 
 # FIXME get these from commandline (and validate 'em)
+savecache = False
 conf['releasever']='10'
 conf['basearch']='i386'
 
-repolist=('fedora','fedora-updates','fedora-updates-testing')
+#repolist=('fedora','fedora-updates','fedora-updates-testing')
+repolist=('fedora-updates-testing',)
 
 class DebuginfoFSDownloader(YumUtilBase):
     NAME = 'debuginfofs-mirror'
@@ -56,46 +61,148 @@ class DebuginfoFSDownloader(YumUtilBase):
                              DebuginfoFSDownloader.USAGE)
         self.logger = logging.getLogger("yum.verbose.cli.debuginfofs")
 
-y = DebuginfoFSDownloader()
-y.doConfigSetup()
-# TODO check for RepoError
-# Kinda stupid, setting up and then immediately tearing down all repos..
-y.repos.disableRepo('*')
-# No yum cache needed, since we're saving the packages ourselves
-y.repos.setCacheDir(conf['cachedir'])
-y.conf.cache = 0
-y.repos.setCache(0)
-# TODO: optionally allow caching?
-# kind of a cheap hack
-my_yumvar = y.conf.yumvar
-my_yumvar.update(conf)
+def yum_setup():
+    y = DebuginfoFSDownloader()
+    y.doConfigSetup()
+    # TODO check for RepoError
+    # Kinda stupid, setting up and then immediately tearing down all repos..
+    y.repos.disableRepo('*')
+    # No yum cache needed, since we're saving the packages ourselves
+    y.repos.setCacheDir(conf['cachedir'])
+    y.conf.cache = 0
+    y.repos.setCache(0)
+    # TODO: optionally allow caching?
+    # FIXME repo cache data is ending up in $CWD
+    return y
+
+def fix_perms(targetdir):
+    '''Make all files readable, and all directories read+execute'''
+    for top, dirs, files in os.walk(targetdir):
+        for d in dirs:
+            i = os.path.join(top, d)
+            mode = os.stat(i)[0]
+            os.chmod(i, mode | 0555)
+        for f in files:
+            i = os.path.join(top, f)
+            if not os.path.islink(i):
+                mode = os.stat(i)[0]
+                os.chmod(i, mode | 0444)
+
+def unpack_rpm(rpm, targetdir):
+    created_dir = False
+    if not os.path.isdir(targetdir):
+        os.makedirs(targetdir)
+        created_dir = True
 
-for repoid in repolist:
     try:
-        d = dict([(k,varReplace(v,my_yumvar)) for k,v in c.items(repoid)])
-    except ConfigParser.NoSectionError:
-        print "No section named %s in config" % repoid
-        continue
-    if 'mirrorlist' in d:
-        print "Adding repo %s with mirrorlist %s" % (repoid,d['mirrorlist'])
-        repo = YumRepository(repoid + '-debuginfofs')
-        repo.name = repo.id
-        repo.mirrorlist = d['mirrorlist']
-        repo.cache = 0
-        repo.enable()
-        y.repos.add(repo)
-        y.repos.doSetup(thisrepo=repo.id)
+        # FIXME: unpack into a tmpdir and then move into place after finishing
+        # or else interrupted unpacks will leave us with incomplete output
+        os.chdir(targetdir)
+        # rpm2cpio $rpm | cpio --quiet -iumd
+        p1 = Popen(['rpm2cpio',rpm], stdout=PIPE)
+        p2 = Popen(['cpio','--quiet','-iumd'], stdin=p1.stdout, stdout=PIPE)
+        output = p2.communicate()[0] # should be empty
+        if p2.returncode != 0:
+            raise OSError, "cpio failed: %s output:\n%s" % (str(p2.returncode),output)
+        # Fix perms so all files are readable
+        fix_perms(targetdir)
+    except (IOError, OSError), e:
+        print str(e)
+        if created_dir:
+            print "removing %s" % targetdir
+            shutil.rmtree(targetdir)
+        return False
+    return True
+
+def mkdebuginfolinks(sourcedir, targetdir):
+    '''hardlink debuginfo from sourcedir into targetdir'''
+    count = 0
+    for top, dirs, files in os.walk(sourcedir, topdown=True):
+        if '/usr/lib/debug/.build-id/' not in top:
+            continue
+        for u in [os.path.join(top,f) for f in files if f.endswith('.debug')]:
+            target = os.path.realpath(u)
+            linkname = u.split('/usr/lib/debug/.build-id/')[1]
+            newlink = os.path.join(targetdir,linkname)
+            try:
+                os.makedirs(os.path.dirname(newlink))
+            except OSError, e:
+                if e.errno != 17:
+                    raise e
+            if os.path.exists(newlink):
+                os.unlink(newlink)
+            os.link(target,newlink)
+            count += 1
+    return count
+
+# Check permissions
+perms_ok = True
+for d in (conf['exportdir'], conf['cachedir']):
+    if not os.access(d,os.W_OK):
+        perms_ok = False
+        print "ERROR: you don't have write permission on %s" % d
+if not perms_ok:
+    sys.exit(1)
 
+if __name__ == '__main__':
+    y = yum_setup()
+    # kind of a cheap hack
+    my_yumvar = y.conf.yumvar
+    my_yumvar.update(conf)
+    
+    # Uses y, conf, c, repolist
+    for repoid in repolist:
+        try:
+            d = dict([(k,varReplace(v,my_yumvar)) for k,v in c.items(repoid)])
+        except ConfigParser.NoSectionError:
+            print "No section named %s in config" % repoid
+            continue
+        if 'mirrorlist' in d:
+            print "Adding repo %s with mirrorlist %s" % (repoid,d['mirrorlist'])
+            repo = YumRepository(repoid + '-debuginfofs')
+            repo.name = repo.id
+            repo.mirrorlist = d['mirrorlist']
+            repo.cache = 0
+            repo.enable()
+            y.repos.add(repo)
+            y.repos.doSetup(thisrepo=repo.id)
+    
+    print "Downloading to cachedir %s" % conf['cachedir']
+    print "Unpacking to exportdir %s" % conf['exportdir']
     packages = y.pkgSack.returnPackages()
+    
     for p in packages:
         repo = y.repos.getRepo(p.repoid)
         remote = p.returnSimple('relativepath')
         local = os.path.join(conf['cachedir'],os.path.basename(remote))
         p.localpath = local
-        # TODO: Check to see if we've already got it in cache
-        # TODO: Check to see if it's already unpacked
-        # Download
-        #path = repo.getPackage(p)
-        # TODO: Unpack 
-        # TODO: Remove cached package
-        # TODO: Hardlink
+        (n,a,e,v,r) = p.pkgtup
+        nevra = "%s-%s:%s-%s.%s" % (n,e,v,r,a)
+        #/var/www/debuginfofs/packages/c/coreutils/coreutils-0:6.12-18.fc10.i386
+        newdir = os.path.join(n[0],n,nevra)
+        targetdir = os.path.join(conf['exportdir'],'packages',newdir)
+        # Check to see if it's already unpacked
+        if os.path.isdir(targetdir):
+            print "Already unpacked %s" % nevra
+            # TODO optionally rmtree and redownload/unpack
+            # FIXME: continue, or just skip fetch/unpack and do links anyway?
+            continue
+    
+        # Check to see if we've already got it in cache
+        if (os.path.exists(local) and
+            os.path.getsize(local) == int(p.returnSimple('packagesize'))):
+            print "Already downloaded %s" % nevra
+        else:
+            # Download package
+            print "Downloading %s" % nevra
+            # FIXME: uninterruptible!
+            path = repo.getPackage(p)
+    
+        print "Unpacking to %s" % newdir
+        unpack_rpm(local, targetdir)
+        # Remove cached package now that we've unpacked it
+        if not savecache:
+            os.unlink(local)
+        # Make hardlinks
+        r=mkdebuginfolinks(targetdir,os.path.join(conf['exportdir'],'build-id'))
+        print "Linked %i debuginfo file%s" % (r, r != 1 and "s" or "")