path: root/debuginfofs-mirror

#!/usr/bin/python
# Given Fedora arch and version, fetch all debuginfo packages from the mirrors
# Copyright 2009 Red Hat, Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
# 
# Author: Will Woods <wwoods@redhat.com>

import os
import sys
import shutil
import logging
import optparse
import ConfigParser
import rpmUtils.arch
# Why, yes, I do like to arrange imports by line length
from subprocess import Popen, PIPE
from yum import _
import yum.Errors
from yum.parser import varReplace
from yum.yumRepo import YumRepository
sys.path.insert(0,'/usr/share/yum-cli')
from utils import YumUtilBase

configfile = '/etc/debuginfofs.conf'
c = ConfigParser.RawConfigParser()
try:
    c.read(configfile)
    conf = dict(c.items('general'))
except (IOError, 
        ConfigParser.MissingSectionHeaderError, 
        ConfigParser.NoSectionError):
    print "Error: no 'cachedir' in %s" % configfile
    sys.exit(1)

class DebuginfoFSDownloader(YumUtilBase):
    NAME = 'debuginfofs-mirror'
    VERSION = '0.1'
    USAGE = 'usage: %s REPOID [REPOID..]' % NAME
    def __init__(self):
        YumUtilBase.__init__(self,
                             DebuginfoFSDownloader.NAME,
                             DebuginfoFSDownloader.VERSION,
                             DebuginfoFSDownloader.USAGE)
        self.optparser = self.getOptionParser()

    def modify_options(self):
        o = self.optparser
        # FIXME: remove some inappropriate options (--enablerepo etc)
        o.add_option("--cachedir", default=conf['cachedir'],
                help="directory to use for caching RPMs and metadata")
        o.add_option("--exportdir", default=conf['exportdir'],
                help="toplevel directory for debuginfofs exported data")
        o.add_option("--savecache", default=False, action="store_true",
                help="don't delete downloaded RPMs after unpacking them")
        # TODO
        #o.add_option("--distro", default="Fedora",
        #        help="Use debuginfo repos for this distro.")
        # TODO: validate releasever/basearch
        o.add_option("--releasever", default="10",
                help="download debuginfo for this version of the distro.")
        o.add_option("--basearch", default="i386",
                help="download debuginfo for this arch")

    def setup(self):
        self.modify_options()
        try:
            # sets self.cmds
            opt = self.doUtilConfigSetup()
        except yum.Errors.RepoError, e:
            self.logger.error(str(e))
            sys.exit(50)
        # Kinda stupid, setting up and then immediately tearing down all repos..
        self.repos.disableRepo('*')
        self.repos.setCacheDir(opt.cachedir)
        # No yum cache needed, since we're saving the packages ourselves
        self.conf.cache = 0
        self.repos.setCache(0)

        return opt

    def listDownloads(self, packages):
        '''Returns a string representation of the packages to be downloaded'''
        n_wid = s_wid = 1
        coldata = []
        for p in packages:
            nevra = pkg_to_nevra(p)
            size = self.format_number(int(p.returnSimple('packagesize')))
            if len(nevra) > n_wid:
                n_wid = len(nevra)
            if len(size) > s_wid:
                s_wid = len(size)
            coldata.append((nevra, size))
        hr = '=' * self.term.columns
        head = self.fmtColumns(((_('Package'),-n_wid),(_('Size'),-s_wid)), u" ")
        out = u"""%s\n%s\n%s""" % (hr, head, hr)
        for nevra, size in coldata:
            cols = ((nevra,-n_wid),(size, s_wid))
            line = self.fmtColumns(cols, u" ", u"\n")
            out += line
        return out

def pkg_to_nevra(p):
    '''return NEVRA string for package object p'''
    (n,a,e,v,r) = p.pkgtup
    return "%s-%s:%s-%s.%s" % (n,e,v,r,a)

def fix_perms(targetdir):
    '''Make all files readable, and all directories read+execute'''
    for top, dirs, files in os.walk(targetdir):
        for d in dirs:
            i = os.path.join(top, d)
            mode = os.stat(i)[0]
            os.chmod(i, mode | 0555)
        for f in files:
            i = os.path.join(top, f)
            if not os.path.islink(i):
                mode = os.stat(i)[0]
                os.chmod(i, mode | 0444)

def unpack_rpm(rpm, targetdir):
    created_dir = False
    if not os.path.isdir(targetdir):
        os.makedirs(targetdir)
        created_dir = True

    try:
        # FIXME: unpack into a tmpdir and then move into place after finishing
        # or else interrupted unpacks will leave us with incomplete output
        os.chdir(targetdir)
        # rpm2cpio $rpm | cpio --quiet -iumd
        p1 = Popen(['rpm2cpio',rpm], stdout=PIPE)
        p2 = Popen(['cpio','--quiet','-iumd'], stdin=p1.stdout, stdout=PIPE)
        output = p2.communicate()[0] # should be empty
        if p2.returncode != 0:
            raise OSError, "cpio failed: %s output:\n%s" % (str(p2.returncode),output)
        # Fix perms so all files are readable
        fix_perms(targetdir)
    except (IOError, OSError), e:
        print str(e)
        if created_dir:
            print "removing %s" % targetdir
            shutil.rmtree(targetdir)
        return False
    return True

def mkdebuginfolinks(sourcedir, targetdir):
    '''hardlink debuginfo from sourcedir into targetdir'''
    count = 0
    for top, dirs, files in os.walk(sourcedir, topdown=True):
        if '/usr/lib/debug/.build-id/' not in top:
            continue
        for u in [os.path.join(top,f) for f in files if f.endswith('.debug')]:
            target = os.path.realpath(u)
            linkname = u.split('/usr/lib/debug/.build-id/')[1]
            newlink = os.path.join(targetdir,linkname)
            try:
                os.makedirs(os.path.dirname(newlink))
            except OSError, e:
                if e.errno != 17:
                    raise e
            if os.path.exists(newlink):
                os.unlink(newlink)
            os.link(target,newlink)
            count += 1
    return count

if __name__ == '__main__':
    y = DebuginfoFSDownloader()
    opt = y.setup()
    # FIXME handle KeyboardInterrupt
    # FIXME this is stupid - figure out how --verbose works
    if opt.verbose:
        y.logger = y.verbose_logger
    # Validate some of the opts
    # Check permissions on exportdir/cachedir
    perms_ok = True
    for d in (opt.exportdir, opt.cachedir):
        if not os.access(d,os.W_OK):
            perms_ok = False
            y.logger.error("you don't have write permission on %s" % d)
    if not perms_ok:
        sys.exit(1)

    if opt.basearch not in rpmUtils.arch.arches:
        y.logger.error("Unrecognized arch %s" % opt.basearch)
        sys.exit(1)

    # kind of a cheap hack - use values from conf and opt to re-parse URLs
    my_yumvar = y.conf.yumvar
    my_yumvar.update(conf)
    for a in 'basearch', 'releasever':
        v = getattr(opt,a,None)
        if v:
            my_yumvar[a] = v
    # Another cheap hack - lie about our arch
    rpmUtils.arch.canonArch = opt.basearch

    # Set up the requested repos
    for repoid in y.cmds:
        try:
            d = dict([(k,varReplace(v,my_yumvar)) for k,v in c.items(repoid)])
        except ConfigParser.NoSectionError:
            y.logger.error("Unknown repo ID %s" % repoid)
            continue
        if 'mirrorlist' in d:
            y.logger.info("Adding repo %s with mirrorlist %s" % (repoid,d['mirrorlist']))
            repo = YumRepository('%s-%s-%s-debuginfofs' % \
                                 (repoid, opt.releasever, opt.basearch))
            repo.name = repo.id
            repo.mirrorlist = d['mirrorlist']
            repo.cache = 0
            repo.enable()
            repo.basecachedir = opt.cachedir
            repo.cachedir = repo.id
            y.repos.add(repo)
            y.repos.doSetup(thisrepo=repo.id)

    repolist = y.repos.listEnabled()
    if repolist:
        # Set up progress callbacks for new repos
        y.setupProgressCallbacks()
        # STUPID EXCLUDEARCH
        archset = set()
        for sack in y.pkgSack.sacks.values():
            archset = archset.union(sack._arch_allowed)
        archset.add(opt.basearch)
        y.pkgSack.excludeArchs(archset)
        for r in repolist:
            try:
                r.ready()
            except yum.Errors.RepoError, e:
                y.logger.error(str(e))
                sys.exit(1)
    else:
        y.logger.error("Couldn't find any repos to set up.")
        repos = [s for s in c.sections() if s != 'general']
        y.logger.error("Known repos: %s" % ", ".join(repos))
        sys.exit(1)
    
    # Figure out which packages we need to fetch
    needpackages = []
    for p in y.pkgSack.returnPackages():
        repo = y.repos.getRepo(p.repoid)
        remote = p.returnSimple('relativepath')
        local = os.path.join(opt.cachedir,os.path.basename(remote))
        p.localpath = local
        n = p.pkgtup[0]
        nevra = pkg_to_nevra(p)
        #/var/www/debuginfofs/packages/c/coreutils/coreutils-0:6.12-18.fc10.i386
        newdir = os.path.join(n[0],n,nevra)
        targetdir = os.path.join(opt.exportdir,'packages',newdir)
        # Check to see if it's already unpacked
        if os.path.isdir(targetdir):
            y.logger.info("Already unpacked %s" % nevra)
            # TODO optionally rmtree and redownload/unpack
            # FIXME: continue, or just skip fetch/unpack and do links anyway?
            continue
    
        # Check to see if we've already got it in cache
        # XXX does y.downloadPackages handle this for us?
        if (os.path.exists(local) and
            os.path.getsize(local) == int(p.returnSimple('packagesize'))):
            y.logger.info("Already downloaded %s" % nevra)
        else:
            needpackages.append(p)

    if not needpackages:
        y.verbose_logger.info(_('Nothing to do!'))
        sys.exit(0)

    # Show what will be downloaded and get confirmation
    y.verbose_logger.info(y.listDownloads(needpackages))
    y.reportDownloadSize(needpackages)
    y.verbose_logger.info("Downloading to cachedir %s" % opt.cachedir)
    y.verbose_logger.info("Unpacking to exportdir %s" % opt.exportdir)
    if y._promptWanted():
        if not y.userconfirm():
            y.logger.info(_('Exiting on user Command'))
            sys.exit(1)

    # Download packages
    y.downloadPkgs(needpackages, callback_total=y.download_callback_total_cb)
    
    # Unpack and hardlink downloaded RPMs
    for p in needpackages:
        local = p.localpath
        (n,a,e,v,r) = p.pkgtup
        nevra = "%s-%s:%s-%s.%s" % (n,e,v,r,a)
        newdir = os.path.join(n[0],n,nevra)
        targetdir = os.path.join(opt.exportdir,'packages',newdir)

        y.verbose_logger.info("Unpacking %s" % nevra)
        unpack_rpm(local, targetdir)
        # Remove cached package now that we've unpacked it
        if not opt.savecache:
            os.unlink(local)
        # Make hardlinks
        r = mkdebuginfolinks(targetdir, os.path.join(opt.exportdir,'build-id'))
        y.logger.info("Linked %3i debuginfo file%s" % (r, r != 1 and "s" or ""))