path: root/src/plugins/abrt-action-install-debuginfo

#! /usr/bin/python -u
# -*- coding: utf-8 -*-
# WARNING: python -u means unbuffered I/O without it the messages are
# passed to the parent asynchronously which looks bad in clients.

PROGNAME = "abrt-action-install-debuginfo"

import sys
import os
import errno
import time
import getopt
import shutil
from subprocess import Popen, PIPE
from yum import _, YumBase
from yum.callbacks import DownloadBaseCallback

# everything was ok
RETURN_OK = 0
# serious problem, should be logged somewhere
RETURN_FAILURE = 2


GETTEXT_PROGNAME = "abrt"
import locale
import gettext

_ = lambda x: gettext.lgettext(x)

def init_gettext():
    try:
        locale.setlocale(locale.LC_ALL, "")
    except locale.Error:
        os.environ['LC_ALL'] = 'C'
        locale.setlocale(locale.LC_ALL, "")
    gettext.bind_textdomain_codeset(GETTEXT_PROGNAME, locale.nl_langinfo(locale.CODESET))
    gettext.bindtextdomain(GETTEXT_PROGNAME, '/usr/share/locale')
    gettext.textdomain(GETTEXT_PROGNAME)


verbose = 0
def log(fmt, *args):
    sys.stderr.write("%s\n" % (fmt % args))

def log1(fmt, *args):
    """ prints log message if verbosity >= 1 """
    if verbose >= 1:
        sys.stderr.write("%s\n" % (fmt % args))

def log2(fmt, *args):
    """ prints log message if verbosity >= 2 """
    if verbose >= 2:
        sys.stderr.write("%s\n" % (fmt % args))

def error_msg(fmt, *args):
    sys.stderr.write("%s\n" % (fmt % args))

def error_msg_and_die(fmt, *args):
    sys.stderr.write("%s\n" % (fmt % args))
    sys.exit(1)


old_stdout = -1
def mute_stdout():
    if verbose < 2:
        global old_stdout
        old_stdout = sys.stdout
        sys.stdout = open("/dev/null", "w")

def unmute_stdout():
    if verbose < 2:
        if old_stdout != -1:
            sys.stdout = old_stdout
        else:
            print "ERR: unmute called without mute?"

def ask_yes_no(prompt, retries=4):
    while True:
        try:
            response = raw_input(prompt)
        except EOFError:
            log1("got eof, probably executed from helper, assuming - yes")
            return True
        if response in (_("y")): # for translators -> y/Y as yes
            return True
        if response in ("", _("n")): # for translators -> N/n as no
            return False
        retries = retries - 1
        if retries < 0:
            break
    return False

# TODO: unpack just required debuginfo and not entire rpm?
# ..that can lead to: foo.c No such file and directory
# files is not used...
def unpack_rpm(package_file_name, files, tmp_dir, destdir, keeprpm):
    package_full_path = tmp_dir + "/" + package_file_name
    log1("Extracting %s to %s", package_full_path, destdir)
    log2("%s", files)
    print _("Extracting cpio from %s") % (package_full_path)
    unpacked_cpio_path = tmp_dir + "/unpacked.cpio"
    try:
        unpacked_cpio = open(unpacked_cpio_path, 'wb')
    except IOError, ex:
        print _("Can't write to '%s': %s") % (unpacked_cpio_path, ex)
        return RETURN_FAILURE
    rpm2cpio = Popen(["rpm2cpio", package_full_path],
                       stdout = unpacked_cpio, bufsize = -1)
    retcode = rpm2cpio.wait()

    if retcode == 0:
        log1("cpio written OK")
        if not keeprpm:
            log1("keeprpms = False, removing %s", package_full_path)
            #print _("Removing temporary rpm file")
            os.unlink(package_full_path)
    else:
        unpacked_cpio.close()
        print _("Can't extract package '%s'") % package_full_path
        return RETURN_FAILURE

    # close the file
    unpacked_cpio.close()
    # and open it for reading
    unpacked_cpio = open(unpacked_cpio_path, 'rb')

    print _("Caching files from %s made from %s") % ("unpacked.cpio", package_file_name)
    cpio = Popen(["cpio", "-idu", "--quiet"],
                  stdin=unpacked_cpio, cwd=destdir, bufsize=-1)
    retcode = cpio.wait()

    if retcode == 0:
        log1("files extracted OK")
        #print _("Removing temporary cpio file")
        os.unlink(unpacked_cpio_path)
    else:
        print _("Can't extract files from '%s'") % unpacked_cpio_path
        return RETURN_FAILURE

class MyDownloadCallback(DownloadBaseCallback):
    def __init__(self, total_pkgs):
        self.total_pkgs = total_pkgs
        self.downloaded_pkgs = 0
        self.last_pct = 0
        self.last_time = 0
        DownloadBaseCallback.__init__(self)

    def updateProgress(self, name, frac, fread, ftime):
        pct = int(frac * 100)
        if pct == self.last_pct:
            log2("percentage is the same, not updating progress")
            return

        self.last_pct = pct
        # if run from terminal we can have fancy output
        if sys.stdout.isatty():
            sys.stdout.write("\033[sDownloading (%i of %i) %s: %3u%%\033[u"
                    % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct)
            )
            if pct == 100:
                print (_("Downloading (%i of %i) %s: %3u%%")
                        % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct)
                )
        # but we want machine friendly output when spawned from abrt-server
        else:
            t = time.time()
            if self.last_time == 0:
                self.last_time = t
            # update only every 10 seconds
            if pct == 100 or self.last_time > t or t - self.last_time >= 10:
                print (_("Downloading (%i of %i) %s: %3u%%")
                        % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct)
                )
                self.last_time = t
                if pct == 100:
                    self.last_time = 0

        sys.stdout.flush()

class DebugInfoDownload(YumBase):
    def __init__(self, cache, tmp, keep_rpms=False):
        self.cachedir = cache
        self.tmpdir = tmp
        self.keeprpms = keep_rpms
        YumBase.__init__(self)
        mute_stdout()
        #self.conf.cache = os.geteuid() != 0
        # Setup yum (Ts, RPM db, Repo & Sack)
        try:
            # Saw this exception here:
            # cannot open Packages index using db3 - Permission denied (13)
            # yum.Errors.YumBaseError: Error: rpmdb open failed
            self.doConfigSetup()
        except Exception, e:
            unmute_stdout()
            print _("Error initializing yum (YumBase.doConfigSetup): '%s'") % str(e)
            #return 1 - can't do this in constructor
            exit(1)
        unmute_stdout()

    # return value will be used as exitcode. So 0 = ok, !0 - error
    def download(self, files):
        """ @files - """
        installed_size = 0
        total_pkgs = 0
        todownload_size = 0
        downloaded_pkgs = 0
        # nothing to download?
        if not files:
            return

        #if verbose == 0:
        #    # this suppress yum messages about setting up repositories
        #    mute_stdout()

        # make yumdownloader work as non root user
        if not self.setCacheDir():
            self.logger.error("Error: can't make cachedir, exiting")
            exit(50)

        # disable all not needed
        for repo in self.repos.listEnabled():
            repo.close()
            self.repos.disableRepo(repo.id)
        # enable -debuginfo repos
        for repo in self.repos.findRepos(pattern="*debuginfo*"):
            #print repo
            repo.enable()
            rid = self.repos.enableRepo(repo.id)
            log1("enabled repo %s", rid)
            setattr(repo, "skip_if_unavailable", True)

        self.repos.doSetup()

        # This is somewhat "magic", it unpacks the metadata making it usable.
        # Looks like this is the moment when yum talks to remote servers,
        # which takes time (sometimes minutes), let user know why
        # we have "paused":
        print _("Looking for needed packages in repositories")
        try:
            self.repos.populateSack(mdtype='metadata', cacheonly=1)
        except Exception, e:
            print _("Error retrieving metadata: '%s'") % str(e)
            return 1
        try:
            # Saw this exception here:
            # raise Errors.NoMoreMirrorsRepoError, errstr
            # NoMoreMirrorsRepoError: failure:
            # repodata/7e6632b82c91a2e88a66ad848e231f14c48259cbf3a1c3e992a77b1fc0e9d2f6-filelists.sqlite.bz2
            # from fedora-debuginfo: [Errno 256] No more mirrors to try.
            self.repos.populateSack(mdtype='filelists', cacheonly=1)
        except Exception, e:
            print _("Error retrieving filelists: '%s'") % str(e)
            return 1

        #if verbose == 0:
        #    # re-enable the output to stdout
        #    unmute_stdout()

        not_found = []
        package_files_dict = {}
        for debuginfo_path in files:
            log2("yum whatprovides %s", debuginfo_path)
            pkg = self.pkgSack.searchFiles(debuginfo_path)
            # sometimes one file is provided by more rpms, we can use either of
            # them, so let's use the first match
            if pkg:
                if pkg[0] in package_files_dict.keys():
                    package_files_dict[pkg[0]].append(debuginfo_path)
                else:
                    package_files_dict[pkg[0]] = [debuginfo_path]
                    todownload_size += float(pkg[0].size)
                    installed_size += float(pkg[0].installedsize)
                    total_pkgs += 1

                log2("found pkg for %s: %s", debuginfo_path, pkg[0])
            else:
                log2("not found pkg for %s", debuginfo_path)
                not_found.append(debuginfo_path)

        # connect our progress update callback
        dnlcb = MyDownloadCallback(total_pkgs)
        self.repos.setProgressBar(dnlcb)

	if verbose != 0 or len(not_found) != 0:
	    print _("Can't find packages for %u debuginfo files") % len(not_found)
	if verbose != 0 or total_pkgs != 0:
	    print _("Packages to download: %u") % total_pkgs
	    print _("Downloading %.2fMb, installed size: %.2fMb") % (
	             todownload_size / (1024**2),
	             installed_size / (1024**2)
	            )

        # ask only if we have terminal, because for now we don't have a way
        # how to pass the question to gui and the response back
        if noninteractive == False and sys.stdout.isatty():
            if not ask_yes_no(_("Is this ok? [y/N] ")):
                return RETURN_OK

        for pkg, files in package_files_dict.iteritems():
            dnlcb.downloaded_pkgs = downloaded_pkgs
            repo.cache = 0
            remote = pkg.returnSimple('relativepath')
            local = os.path.basename(remote)
            if not os.path.exists(self.tmpdir):
                os.makedirs(self.tmpdir)
            if not os.path.exists(self.cachedir):
                os.makedirs(self.cachedir)
            local = os.path.join(self.tmpdir, local)
            pkg.localpath = local # Hack: to set the localpath we want
            err = self.downloadPkgs(pkglist=[pkg])
            # normalize the name
            # just str(pkg) doesn't work because it can have epoch
            pkg_nvra = pkg.name + "-" + pkg.version + "-" + pkg.release + "." + pkg.arch
            package_file_name = pkg_nvra + ".rpm"
            if err:
                # I observed a zero-length file left on error,
                # which prevents cleanup later. Fix it:
                try:
                    os.unlink(self.tmpdir + "/" + package_file_name)
                except:
                    pass
                print (_("Downloading package %s failed") % pkg)
            else:
                unpack_result = unpack_rpm(package_file_name, files, self.tmpdir,
                                           self.cachedir, keeprpms)
                if unpack_result == RETURN_FAILURE:
                    # recursively delete the temp dir on failure
                    print _("Unpacking failed, aborting download...")
                    clean_up()
                    return RETURN_FAILURE

            downloaded_pkgs += 1

        if not self.keeprpms and os.path.exists(self.tmpdir):
            # Was: "All downloaded packages have been extracted, removing..."
            # but it was appearing even if no packages were in fact extracted
            # (say, when there was one package, and it has download error).
            print (_("Removing %s") % self.tmpdir)
            try:
                os.rmdir(self.tmpdir)
            except OSError:
                error_msg(_("Can't remove %s, probably contains an error log"), self.tmpdir)

def build_ids_to_path(build_ids):
    """
    build_id1=${build_id:0:2}
    build_id2=${build_id:2}
    file="usr/lib/debug/.build-id/$build_id1/$build_id2.debug"
    """
    return ["/usr/lib/debug/.build-id/%s/%s.debug" % (b_id[:2], b_id[2:]) for b_id in build_ids]

# beware this finds only missing libraries, but not the executable itself ..

def filter_installed_debuginfos(build_ids, cache_dir):
    # 1st pass -> search in /usr/lib
    missing_di = []
    files = build_ids_to_path(build_ids)
    for debuginfo_path in files:
        cache_debuginfo_path = cache_dir + debuginfo_path
        log2("checking path: %s", debuginfo_path)
        if os.path.exists(debuginfo_path):
            log2("found: %s", debuginfo_path)
            continue
        if os.path.exists(cache_debuginfo_path):
            log2("found: %s", cache_debuginfo_path)
            continue
        log2("not found: %s", cache_debuginfo_path)
        missing_di.append(debuginfo_path)
    return missing_di

tmpdir = None
def clean_up():
    if tmpdir:
        try:
            shutil.rmtree(tmpdir)
        except OSError, ex:
            if ex.errno != errno.ENOENT:
                error_msg(_("Can't remove '%s': %s"), tmpdir, ex)

def sigterm_handler(signum, frame):
    clean_up()
    exit(RETURN_OK)

def sigint_handler(signum, frame):
    clean_up()
    print "\n", _("Exiting on user command")
    sys.stdout.flush()
    # ??! without "sys.", I am getting segv!
    sys.exit(RETURN_OK)

import signal

if __name__ == "__main__":
    # abrt-server can send SIGTERM to abort the download
    signal.signal(signal.SIGTERM, sigterm_handler)
    # ctrl-c
    signal.signal(signal.SIGINT, sigint_handler)
    fbuild_ids = "build_ids"
    cachedir = None
    size_mb = 4096
    tmpdir = None
    keeprpms = False
    noninteractive = False
    b_ids = []

    # localization
    init_gettext()

    ABRT_VERBOSE = os.getenv("ABRT_VERBOSE")
    if (ABRT_VERBOSE):
        try:
            verbose = int(ABRT_VERBOSE)
        except:
            pass

    PROGNAME = os.path.basename(sys.argv[0])
    # ____________________________________________________________________________________ 7
    # ______01234567890123456789012345678901234567890123456789012345678901234567890123456789
    help_text = _(
            "Usage: %s [-vy] [--ids=BUILD_IDS_FILE]\n"
            "       [--tmpdir=TMPDIR] [--cache=CACHEDIR] [--size_mb=SIZE]\n"
            "\n"
            "Installs debuginfos for all build-ids listed in BUILD_IDS_FILE\n"
            "to CACHEDIR, using TMPDIR as temporary staging area.\n"
            "Old files in CACHEDIR are deleted until it is smaller than SIZE.\n"
            "\n"
            "    -v          Be verbose\n"
            "    -y          Noninteractive, assume 'Yes' to all questions\n"
            "    --ids       Default: build_ids\n"
            "    --tmpdir    Default: /tmp/abrt-tmp-debuginfo-RANDOM_SUFFIX\n"
            "    --cache     Default: /var/cache/abrt-di\n"
            "    --size_mb   Default: 4096\n"
            # --keeprpms is not documented yet because it's a NOP so far
    ) % PROGNAME

    try:
        opts, args = getopt.getopt(sys.argv[1:], "vyh",
                ["help", "ids=", "cache=", "size_mb=", "tmpdir=", "keeprpms"])
    except getopt.GetoptError, err:
        print str(err) # prints something like "option -a not recognized"
        exit(RETURN_FAILURE)

    for opt, arg in opts:
        if opt in ("-h", "--help"):
            print help_text
            exit(0)
        elif opt == "-v":
            verbose += 1
        elif opt == "-y":
            noninteractive = True
        elif opt == "--ids":
            fbuild_ids = arg
        elif opt == "--cache":
            cachedir = arg
        elif opt == "--size_mb":
            try:
                size_mb = int(arg)
            except:
                pass
        elif opt == "--tmpdir":
            tmpdir = arg
        elif opt == "--keeprpms":
            keeprpms = True

    if not cachedir:
        cachedir = "/var/cache/abrt-di"
    if not tmpdir:
        # security people prefer temp subdirs in app's private dir, like /var/run/abrt
        # for now, we use /tmp...
        tmpdir = "/tmp/abrt-tmp-debuginfo-%s.%u" % (time.strftime("%Y-%m-%d-%H:%M:%S"), os.getpid())

    fin = sys.stdin
    if fbuild_ids != "-":
        try:
            fin = open(fbuild_ids, "r")
        except IOError, ex:
            error_msg_and_die(_("Can't open %s: %s"), fbuild_ids, ex)
    for line in fin.readlines():
        b_ids.append(line.strip('\n'))

    if not b_ids:
        exit(RETURN_FAILURE)

    # Delete oldest/biggest files from cachedir.
    # (Note that we need to do it before we check for missing debuginfos)
    #
    # We can do it as a separate step in abrt_event.conf, but this
    # would require setuid'ing abrt-action-trim-files to abrt:abrt.
    # Since we (via abrt-action-install-debuginfo-to-abrt-cache)
    # are already running setuid,
    # it makes sense to NOT setuid abrt-action-trim-files too,
    # but instead run it as our child:
    sys.stdout.flush()
    try:
        pid = os.fork()
        if pid == 0:
            log1("abrt-action-trim-files -f %um:%s", size_mb, cachedir);
            os.execlp("abrt-action-trim-files", "abrt-action-trim-files", "-f", "%um:%s" % (size_mb, cachedir));
        if pid > 0:
            os.waitpid(pid, 0);
    except Exception, e:
        error_msg("Can't execute abrt-action-trim-files: %s", e);

    missing = filter_installed_debuginfos(b_ids, cachedir)
    if missing:
        log2("%s", missing)
        print _("Coredump references %u debuginfo files, %u of them are not installed") % (len(b_ids), len(missing))

        # TODO: should we pass keep_rpms=keeprpms to DebugInfoDownload here??
        downloader = DebugInfoDownload(cache=cachedir, tmp=tmpdir)
        try:
            result = downloader.download(missing)
        except Exception, ex:
            error_msg_and_die("Can't download debuginfos: %s", ex)

        missing = filter_installed_debuginfos(b_ids, cachedir)
        for bid in missing:
            print _("Missing debuginfo file: %s") % bid
        exit(result)

    print _("All %u debuginfo files are available") % len(b_ids)
    exit(RETURN_OK)