debuginfofs-mirror basically works as intended; update TODO

2 files changed, 178 insertions, 68 deletions

diff --git a/TODO b/TODO
index 62d77ad..cdcadf9 100644
--- a/TODO
+++ b/TODO
@@ -1,10 +1,10 @@
 For 1.0:
-- write script to fetch all (debuginfo) packages for distro/arch, using:
-  - yum repo config files
-  - rsync URL
-  - nfs URL / local path
-  - etc?
-- rewrite unpack-debuginfo.sh in Python (for importing elsewhere)?
+- debuginfofs-mirror:
+  - Fix half-assed logging
+  - Fix half-assed i18n
+  - Clean up methods (move stuff from __main__ to the class)
+  - Unpack RPMS to a temporary dir, move it into place (atomic create)
+  - Handle KeyboardInterrupt better, jeez
 - RPM package
   - debuginfofs (client)
   - debuginfofs-server 
@@ -15,7 +15,13 @@ For 1.0:
   BUILDID_URL is unreachable
 
 Further work:
+- Cleanups for debuginfofs-mirror:
+  - make --verbose work right, jeez
+  - Figure out if we can make yum handle checking for already-cached files
+- make debuginfofs-mirror able to use rsync?
 - Avahi announce server to local network
+
+Crazy future ideas:
 - Provide source files as well
   - Client-side FUSE filesystem that auto-fills src/ dir as build-id files
     are read
diff --git a/debuginfofs-mirror b/debuginfofs-mirror
index 5370fbe..5993b41 100755
--- a/debuginfofs-mirror
+++ b/debuginfofs-mirror
@@ -24,12 +24,15 @@ import shutil
 import logging
 import optparse
 import ConfigParser
-# Why, yes, I do arrange imports by line length
+import rpmUtils.arch
+# Why, yes, I do like to arrange imports by line length
 from subprocess import Popen, PIPE
-sys.path.insert(0,'/usr/share/yum-cli')
-from utils import YumUtilBase
+from yum import _
+import yum.Errors
 from yum.parser import varReplace
 from yum.yumRepo import YumRepository
+sys.path.insert(0,'/usr/share/yum-cli')
+from utils import YumUtilBase
 
 configfile = '/etc/debuginfofs.conf'
 c = ConfigParser.RawConfigParser()
@@ -42,38 +45,77 @@ except (IOError,
     print "Error: no 'cachedir' in %s" % configfile
     sys.exit(1)
 
-# FIXME get these from commandline (and validate 'em)
-savecache = False
-conf['releasever']='10'
-conf['basearch']='i386'
-
-#repolist=('fedora','fedora-updates','fedora-updates-testing')
-repolist=('fedora-updates-testing',)
-
 class DebuginfoFSDownloader(YumUtilBase):
     NAME = 'debuginfofs-mirror'
     VERSION = '0.1'
-    USAGE = 'usage: %s' % NAME # XXX update this
+    USAGE = 'usage: %s REPOID [REPOID..]' % NAME
     def __init__(self):
         YumUtilBase.__init__(self,
                              DebuginfoFSDownloader.NAME,
                              DebuginfoFSDownloader.VERSION,
                              DebuginfoFSDownloader.USAGE)
-        self.logger = logging.getLogger("yum.verbose.cli.debuginfofs")
+        self.optparser = self.getOptionParser()
 
-def yum_setup():
-    y = DebuginfoFSDownloader()
-    y.doConfigSetup()
-    # TODO check for RepoError
-    # Kinda stupid, setting up and then immediately tearing down all repos..
-    y.repos.disableRepo('*')
-    # No yum cache needed, since we're saving the packages ourselves
-    y.repos.setCacheDir(conf['cachedir'])
-    y.conf.cache = 0
-    y.repos.setCache(0)
-    # TODO: optionally allow caching?
-    # FIXME repo cache data is ending up in $CWD
-    return y
+    def modify_options(self):
+        o = self.optparser
+        # FIXME: remove some inappropriate options (--enablerepo etc)
+        o.add_option("--cachedir", default=conf['cachedir'],
+                help="directory to use for caching RPMs and metadata")
+        o.add_option("--exportdir", default=conf['exportdir'],
+                help="toplevel directory for debuginfofs exported data")
+        o.add_option("--savecache", default=False, action="store_true",
+                help="don't delete downloaded RPMs after unpacking them")
+        # TODO
+        #o.add_option("--distro", default="Fedora",
+        #        help="Use debuginfo repos for this distro.")
+        # TODO: validate releasever/basearch
+        o.add_option("--releasever", default="10",
+                help="download debuginfo for this version of the distro.")
+        o.add_option("--basearch", default="i386",
+                help="download debuginfo for this arch")
+
+    def setup(self):
+        self.modify_options()
+        try:
+            # sets self.cmds
+            opt = self.doUtilConfigSetup()
+        except yum.Errors.RepoError, e:
+            self.logger.error(str(e))
+            sys.exit(50)
+        # Kinda stupid, setting up and then immediately tearing down all repos..
+        self.repos.disableRepo('*')
+        self.repos.setCacheDir(opt.cachedir)
+        # No yum cache needed, since we're saving the packages ourselves
+        self.conf.cache = 0
+        self.repos.setCache(0)
+
+        return opt
+
+    def listDownloads(self, packages):
+        '''Returns a string representation of the packages to be downloaded'''
+        n_wid = s_wid = 1
+        coldata = []
+        for p in packages:
+            nevra = pkg_to_nevra(p)
+            size = self.format_number(int(p.returnSimple('packagesize')))
+            if len(nevra) > n_wid:
+                n_wid = len(nevra)
+            if len(size) > s_wid:
+                s_wid = len(size)
+            coldata.append((nevra, size))
+        hr = '=' * self.term.columns
+        head = self.fmtColumns(((_('Package'),-n_wid),(_('Size'),-s_wid)), u" ")
+        out = u"""%s\n%s\n%s""" % (hr, head, hr)
+        for nevra, size in coldata:
+            cols = ((nevra,-n_wid),(size, s_wid))
+            line = self.fmtColumns(cols, u" ", u"\n")
+            out += line
+        return out
+
+def pkg_to_nevra(p):
+    '''return NEVRA string for package object p'''
+    (n,a,e,v,r) = p.pkgtup
+    return "%s-%s:%s-%s.%s" % (n,e,v,r,a)
 
 def fix_perms(targetdir):
     '''Make all files readable, and all directories read+execute'''
@@ -135,74 +177,136 @@ def mkdebuginfolinks(sourcedir, targetdir):
             count += 1
     return count
 
-# Check permissions
-perms_ok = True
-for d in (conf['exportdir'], conf['cachedir']):
-    if not os.access(d,os.W_OK):
-        perms_ok = False
-        print "ERROR: you don't have write permission on %s" % d
-if not perms_ok:
-    sys.exit(1)
-
 if __name__ == '__main__':
-    y = yum_setup()
-    # kind of a cheap hack
+    y = DebuginfoFSDownloader()
+    opt = y.setup()
+    # FIXME handle KeyboardInterrupt
+    # FIXME this is stupid - figure out how --verbose works
+    if opt.verbose:
+        y.logger = y.verbose_logger
+    # Validate some of the opts
+    # Check permissions on exportdir/cachedir
+    perms_ok = True
+    for d in (opt.exportdir, opt.cachedir):
+        if not os.access(d,os.W_OK):
+            perms_ok = False
+            y.logger.error("you don't have write permission on %s" % d)
+    if not perms_ok:
+        sys.exit(1)
+
+    if opt.basearch not in rpmUtils.arch.arches:
+        y.logger.error("Unrecognized arch %s" % opt.basearch)
+        sys.exit(1)
+
+    # kind of a cheap hack - use values from conf and opt to re-parse URLs
     my_yumvar = y.conf.yumvar
     my_yumvar.update(conf)
-    
-    # Uses y, conf, c, repolist
-    for repoid in repolist:
+    for a in 'basearch', 'releasever':
+        v = getattr(opt,a,None)
+        if v:
+            my_yumvar[a] = v
+    # Another cheap hack - lie about our arch
+    rpmUtils.arch.canonArch = opt.basearch
+
+    # Set up the requested repos
+    for repoid in y.cmds:
         try:
             d = dict([(k,varReplace(v,my_yumvar)) for k,v in c.items(repoid)])
         except ConfigParser.NoSectionError:
-            print "No section named %s in config" % repoid
+            y.logger.error("Unknown repo ID %s" % repoid)
             continue
         if 'mirrorlist' in d:
-            print "Adding repo %s with mirrorlist %s" % (repoid,d['mirrorlist'])
-            repo = YumRepository(repoid + '-debuginfofs')
+            y.logger.info("Adding repo %s with mirrorlist %s" % (repoid,d['mirrorlist']))
+            repo = YumRepository('%s-%s-%s-debuginfofs' % \
+                                 (repoid, opt.releasever, opt.basearch))
             repo.name = repo.id
             repo.mirrorlist = d['mirrorlist']
             repo.cache = 0
             repo.enable()
+            repo.basecachedir = opt.cachedir
+            repo.cachedir = repo.id
             y.repos.add(repo)
             y.repos.doSetup(thisrepo=repo.id)
+
+    repolist = y.repos.listEnabled()
+    if repolist:
+        # Set up progress callbacks for new repos
+        y.setupProgressCallbacks()
+        # STUPID EXCLUDEARCH
+        archset = set()
+        for sack in y.pkgSack.sacks.values():
+            archset = archset.union(sack._arch_allowed)
+        archset.add(opt.basearch)
+        y.pkgSack.excludeArchs(archset)
+        for r in repolist:
+            try:
+                r.ready()
+            except yum.Errors.RepoError, e:
+                y.logger.error(str(e))
+                sys.exit(1)
+    else:
+        y.logger.error("Couldn't find any repos to set up.")
+        repos = [s for s in c.sections() if s != 'general']
+        y.logger.error("Known repos: %s" % ", ".join(repos))
+        sys.exit(1)
     
-    print "Downloading to cachedir %s" % conf['cachedir']
-    print "Unpacking to exportdir %s" % conf['exportdir']
-    packages = y.pkgSack.returnPackages()
-    
-    for p in packages:
+    # Figure out which packages we need to fetch
+    needpackages = []
+    for p in y.pkgSack.returnPackages():
         repo = y.repos.getRepo(p.repoid)
         remote = p.returnSimple('relativepath')
-        local = os.path.join(conf['cachedir'],os.path.basename(remote))
+        local = os.path.join(opt.cachedir,os.path.basename(remote))
         p.localpath = local
-        (n,a,e,v,r) = p.pkgtup
-        nevra = "%s-%s:%s-%s.%s" % (n,e,v,r,a)
+        n = p.pkgtup[0]
+        nevra = pkg_to_nevra(p)
         #/var/www/debuginfofs/packages/c/coreutils/coreutils-0:6.12-18.fc10.i386
         newdir = os.path.join(n[0],n,nevra)
-        targetdir = os.path.join(conf['exportdir'],'packages',newdir)
+        targetdir = os.path.join(opt.exportdir,'packages',newdir)
         # Check to see if it's already unpacked
         if os.path.isdir(targetdir):
-            print "Already unpacked %s" % nevra
+            y.logger.info("Already unpacked %s" % nevra)
             # TODO optionally rmtree and redownload/unpack
             # FIXME: continue, or just skip fetch/unpack and do links anyway?
             continue
     
         # Check to see if we've already got it in cache
+        # XXX does y.downloadPackages handle this for us?
         if (os.path.exists(local) and
             os.path.getsize(local) == int(p.returnSimple('packagesize'))):
-            print "Already downloaded %s" % nevra
+            y.logger.info("Already downloaded %s" % nevra)
         else:
-            # Download package
-            print "Downloading %s" % nevra
-            # FIXME: uninterruptible!
-            path = repo.getPackage(p)
+            needpackages.append(p)
+
+    if not needpackages:
+        y.verbose_logger.info(_('Nothing to do!'))
+        sys.exit(0)
+
+    # Show what will be downloaded and get confirmation
+    y.verbose_logger.info(y.listDownloads(needpackages))
+    y.reportDownloadSize(needpackages)
+    y.verbose_logger.info("Downloading to cachedir %s" % opt.cachedir)
+    y.verbose_logger.info("Unpacking to exportdir %s" % opt.exportdir)
+    if y._promptWanted():
+        if not y.userconfirm():
+            y.logger.info(_('Exiting on user Command'))
+            sys.exit(1)
+
+    # Download packages
+    y.downloadPkgs(needpackages, callback_total=y.download_callback_total_cb)
     
-        print "Unpacking to %s" % newdir
+    # Unpack and hardlink downloaded RPMs
+    for p in needpackages:
+        local = p.localpath
+        (n,a,e,v,r) = p.pkgtup
+        nevra = "%s-%s:%s-%s.%s" % (n,e,v,r,a)
+        newdir = os.path.join(n[0],n,nevra)
+        targetdir = os.path.join(opt.exportdir,'packages',newdir)
+
+        y.verbose_logger.info("Unpacking %s" % nevra)
         unpack_rpm(local, targetdir)
         # Remove cached package now that we've unpacked it
-        if not savecache:
+        if not opt.savecache:
             os.unlink(local)
         # Make hardlinks
-        r=mkdebuginfolinks(targetdir,os.path.join(conf['exportdir'],'build-id'))
-        print "Linked %i debuginfo file%s" % (r, r != 1 and "s" or "")
+        r = mkdebuginfolinks(targetdir, os.path.join(opt.exportdir,'build-id'))
+        y.logger.info("Linked %3i debuginfo file%s" % (r, r != 1 and "s" or ""))