new debuginfo install script rewritten in python

- using python alows us to use the yum API, so we can read the progress,
  file sizes, requires disk space, etc.. and seems to be faster the using
  yum --whatprovides + yumdownloader
- it's easier to translate
- we can drop dependency on yum-utils

1 files changed, 472 insertions, 0 deletions

diff --git a/src/plugins/abrt-action-install-debuginfo.py b/src/plugins/abrt-action-install-debuginfo.py
new file mode 100755
index 00000000..b16cc3e9
--- /dev/null
+++ b/src/plugins/abrt-action-install-debuginfo.py
@@ -0,0 +1,472 @@
+#! /usr/bin/python -u
+# -*- coding: utf-8 -*-
+
+# WARNING: python -u means unbuffered I/O without it the messages are
+# passed to the parent asynchronously which looks bad in clients..
+from subprocess import Popen, PIPE
+import sys
+import os
+import getopt
+import shutil
+from yum import _, YumBase
+from yum.callbacks import DownloadBaseCallback
+
+# everything was ok
+RETURN_OK = 0
+# serious problem, should be logged somewhere
+RETURN_FAILURE = 2
+
+
+GETTEXT_PROGNAME = "abrt"
+import locale
+import gettext
+
+_ = lambda x: gettext.lgettext(x)
+
+def init_gettext():
+    try:
+        locale.setlocale(locale.LC_ALL, "")
+    except locale.Error:
+        import os
+        os.environ['LC_ALL'] = 'C'
+        locale.setlocale(locale.LC_ALL, "")
+    gettext.bind_textdomain_codeset(GETTEXT_PROGNAME, locale.nl_langinfo(locale.CODESET))
+    gettext.bindtextdomain(GETTEXT_PROGNAME, '/usr/share/locale')
+    gettext.textdomain(GETTEXT_PROGNAME)
+
+
+old_stdout = -1
+def mute_stdout():
+    if verbose < 2:
+        global old_stdout
+        old_stdout = sys.stdout
+        sys.stdout = open("/dev/null", "w")
+
+def unmute_stdout():
+    if verbose < 2:
+        if old_stdout != -1:
+            sys.stdout = old_stdout
+        else:
+            print "ERR: unmute called without mute?"
+
+def ask_yes_no(prompt, retries=4):
+    while True:
+        response = raw_input(prompt)
+        if response in ('y', 'Y'):
+            return True
+        if response in ('n', 'N', ''):
+            return False
+        retries = retries - 1
+        if retries < 0:
+            break
+    return False
+
+# TODO: unpack just required debuginfo and not entire rpm?
+# ..that can lead to: foo.c No such file and directory
+# files is not used...
+def unpack_rpm(package_nevra, files, tmp_dir, destdir, keeprpm):
+    package_file_suffix = ".rpm"
+    package_full_path = tmp_dir + "/" + package_nevra + package_file_suffix
+    log1("Extracting %s to %s" % (package_full_path, destdir))
+    log2(files)
+    print (_("Extracting cpio from %s") % (package_full_path))
+    unpacked_cpio_path = tmp_dir + "/unpacked.cpio"
+    try:
+        unpacked_cpio = open(unpacked_cpio_path, 'wb')
+    except IOError, ex:
+        print (_("Can't write to:"), (unpacked_cpio_path,ex))
+        return RETURN_FAILURE
+    rpm2cpio = Popen(["rpm2cpio", package_full_path],
+                       stdout=unpacked_cpio, bufsize=-1)
+    retcode = rpm2cpio.wait()
+
+    if retcode == 0:
+        log1("cpio written OK")
+        if not keeprpm:
+            log1("keeprpms = False, removing %s" % package_full_path)
+            print _("Removing the temporary rpm file")
+            os.unlink(package_full_path)
+    else:
+        unpacked_cpio.close()
+        print (_("Can't extract package: %s") % package_full_path)
+        return RETURN_FAILURE
+
+    # close the file
+    unpacked_cpio.close()
+    # and open it for reading
+    unpacked_cpio = open(unpacked_cpio_path, 'rb')
+
+    print (_("Caching files from %s made from %s") %
+            (unpacked_cpio_path, package_full_path))
+    cpio = Popen(["cpio","-i", "-d", "--quiet"],
+                  stdin=unpacked_cpio, cwd=destdir, bufsize=-1)
+    retcode = cpio.wait()
+
+    if retcode == 0:
+        log1("files extracted OK")
+        print _("Removing the temporary cpio file")
+        os.unlink(unpacked_cpio_path)
+    else:
+        print (_("Can't extract files from: %s") % unpacked_cpio_path)
+        return RETURN_FAILURE
+
+class MyDownloadCallback(DownloadBaseCallback):
+    def __init__(self, total_pkgs):
+        self.total_pkgs = total_pkgs
+        self.downloaded_pkgs = 0
+        self.last_pct = 0
+        DownloadBaseCallback.__init__(self)
+
+    def updateProgress(self, name, frac, fread, ftime):
+        pct = int( frac*100 )
+        if pct == self.last_pct:
+            log2("percentage is the same, not updating progress")
+            return
+
+        self.last_pct = pct
+        # if run from terminal we can have a fancy output
+        if sys.stdout.isatty():
+            sys.stdout.write("\033[sDownloading (%i of %i) %.30s : %.3s %%\033[u"
+                                % (self.downloaded_pkgs + 1, self.total_pkgs,
+                                    name, pct)
+                            )
+            if pct == 100:
+                print _("Downloading (%i of %i) %.30s : %.3s %%"
+                                % (self.downloaded_pkgs + 1, self.total_pkgs,
+                                    name, pct)
+                        )
+        # but we want machine friendly output when spawned from abrt-server
+        else:
+            print (_("Downloading (%i of %i) %.30s : %.3s %%")
+                      % (self.downloaded_pkgs + 1, self.total_pkgs, name, pct)
+                    )
+
+        sys.stdout.flush()
+
+class DebugInfoDownload(YumBase):
+    """abrt-debuginfo-install --core=CORE tmpdir cachedir"""
+    def __init__(self, cache, tmp, keep_rpms=False):
+        self.cachedir = cache
+        self.tmpdir = tmp
+        self.keeprpms = keep_rpms
+        YumBase.__init__(self)
+        mute_stdout()
+        #self.conf.cache = os.geteuid() != 0
+        # Setup yum (Ts, RPM db, Repo & Sack)
+        self.doConfigSetup()
+        unmute_stdout()
+
+    def download(self, files):
+        """ @files - """
+        installed_size = 0
+        total_pkgs = 0
+        todownload_size = 0
+        downloaded_pkgs = 0
+        # nothing to download?
+        if not files:
+            return
+
+        print _("Searching the missing debuginfo packages")
+        # this suppress yum messages about setting up repositories
+        mute_stdout()
+
+        # make yumdownloader work as non root user.
+        if not self.setCacheDir():
+            self.logger.error("Error: Could not make cachedir, exiting")
+            sys.exit(50)
+
+        # disable all not needed
+        for repo in self.repos.listEnabled():
+            repo.close()
+            self.repos.disableRepo(repo.id)
+        # enable -debuginfo repos
+        for repo in self.repos.findRepos(pattern="*debuginfo*"):
+            #print repo
+            repo.enable()
+            rid = self.repos.enableRepo(repo.id)
+            log1("enabled repo %s" % rid)
+            setattr(repo, "skip_if_unavailable", True)
+        self.repos.doSetup()
+        # This is somewhat "magic", it unpacks the metadata making it usable.
+        self.repos.populateSack(mdtype='metadata', cacheonly=1)
+        self.repos.populateSack(mdtype='filelists', cacheonly=1)
+
+        # re-enable the output to stdout
+        unmute_stdout()
+
+        not_found = []
+        package_files_dict = {}
+        for debuginfo_path in files:
+            log2("yum whatprovides %s" %debuginfo_path)
+            pkg = self.pkgSack.searchFiles(debuginfo_path)
+            # sometimes one file is provided by more rpms, we can use either of
+            # them, so let's use the first match
+            if pkg:
+                if pkg[0] in package_files_dict.keys():
+                    package_files_dict[pkg[0]].append(debuginfo_path)
+                else:
+                    package_files_dict[pkg[0]] = [debuginfo_path]
+                    todownload_size += float(pkg[0].size)
+                    installed_size += float(pkg[0].installedsize)
+                    total_pkgs += 1
+
+                log2("found pkg for %s" % debuginfo_path)
+            else:
+                log2("not found pkg for %s" % debuginfo_path)
+                not_found.append(debuginfo_path)
+
+        # connect our progress update callback
+        dnlcb = MyDownloadCallback(total_pkgs)
+        self.repos.setProgressBar( dnlcb )
+
+        log1("%i files in %i packages" % (len(files), total_pkgs))
+
+        print (_("To download: (%.2f) M / Installed size: %.2f M" %
+                (
+                 todownload_size / (1024**2),
+                 installed_size / (1024**2))
+                )
+              )
+        #print _("%i debug infos not found" % len(not_found))
+
+        log1("packages: %i\nTo download: %f \nUnpacked size: %f" %
+            (total_pkgs,
+             todownload_size / (1024**2),
+             installed_size / (1024**2)))
+
+        # ask only if we have terminal, because for now we don't have a way
+        # how to pass the question to gui and the response back
+        if noninteractive == False and sys.stdout.isatty():
+            if not ask_yes_no(_("Is this ok? [y/N] ")):
+                return RETURN_OK
+
+        for pkg, files in package_files_dict.iteritems():
+            dnlcb.downloaded_pkgs = downloaded_pkgs
+            repo.cache = 0
+            remote = pkg.returnSimple('relativepath')
+            local = os.path.basename(remote)
+            if not os.path.exists(self.tmpdir):
+                os.makedirs(self.tmpdir)
+            if not os.path.exists(self.cachedir):
+                os.makedirs(self.cachedir)
+            local = os.path.join(self.tmpdir, local)
+            pkg.localpath = local # Hack: to set the localpath we want
+            ret = self.downloadPkgs(pkglist=[pkg])
+            # downloadPkgs return an non empty list on succes
+            if ret:
+                print (_("Downloading package %s failed") % pkg)
+            else:
+                # normalize the name
+                # just str(pkg) doesn't work because it can have epoch
+                pkg_nvra = (pkg.name +"-"+ pkg.version +"-"+
+                            pkg.release +"."+ pkg.arch)
+
+                unpack_result = unpack_rpm(pkg_nvra, files, self.tmpdir,
+                                           self.cachedir, keeprpms)
+                if unpack_result == RETURN_FAILURE:
+                    # recursively delete the temp dir on failure
+                    print _("Unpacking failed, aborting download...")
+                    clean_up()
+                    return RETURN_FAILURE
+
+            downloaded_pkgs += 1
+
+        if not self.keeprpms:
+            print (_("All downloaded packages have been extracted, removing %s")
+                 % self.tmpdir)
+            try:
+                os.rmdir(self.tmpdir)
+            except OSError:
+                print _("Can't remove %s, probably contains an error log")
+
+verbose = 0
+def log1(message):
+    """ prints log message if verbosity > 0 """
+    if verbose > 0:
+        print "LOG1:", message
+
+def log2(message):
+    """ prints log message if verbosity > 1 """
+    if verbose > 1:
+        print "LOG2:", message
+
+#eu_unstrip_OUT=`eu-unstrip "--core=$core" -n 2>eu_unstrip.ERR`
+def extract_info_from_core(corefile):
+    """
+    Extracts builds with filenames,
+    Returns a list of tuples (build_id, filename)
+    """
+    #OFFSET = 0
+    BUILD_ID = 1
+    LIBRARY = 2
+    #SEP = 3
+    EXECUTABLE = 4
+
+    print (_("Analyzing corefile: %(corefile_path)s") %
+            {"corefile_path":corefile})
+    eu_unstrip_OUT = Popen(["eu-unstrip","--core=%s" % corefile, "-n"], stdout=PIPE, bufsize=-1).communicate()[0]
+    # parse eu_unstrip_OUT and return the list of build_ids
+
+    # eu_unstrip_OUT = ("0x7f42362ca000+0x204000 c4d35d993598a6242f7525d024b5ec3becf5b447@0x7f42362ca1a0 /usr/lib64/libcanberra-gtk.so.0 - libcanberra-gtk.so.0\n"
+                     # "0x3afa400000+0x210000 607308f916c13c3ad9ee503008d31fa671ba73ce@0x3afa4001a0 /usr/lib64/libcanberra.so.0 - libcanberra.so.0\n"
+                     # "0x3afa400000+0x210000 607308f916c13c3ad9ee503008d31fa671ba73ce@0x3afa4001a0 /usr/lib64/libcanberra.so.0 - libcanberra.so.0\n"
+                     # "0x3bc7000000+0x208000 3be016bb723e85779a23e111a8ab1a520b209422@0x3bc70001a0 /usr/lib64/libvorbisfile.so.3 - libvorbisfile.so.3\n"
+                     # "0x7f423609e000+0x22c000 87f9c7d9844f364c73aa2566d6cfc9c5fa36d35d@0x7f423609e1a0 /usr/lib64/libvorbis.so.0 - libvorbis.so.0\n"
+                     # "0x7f4235e99000+0x205000 b5bc98c125a11b571cf4f2746268a6d3cfa95b68@0x7f4235e991a0 /usr/lib64/libogg.so.0 - libogg.so.0\n"
+                     # "0x7f4235c8b000+0x20e000 f1ff6c8ee30dba27e90ef0c5b013df2833da2889@0x7f4235c8b1a0 /usr/lib64/libtdb.so.1 - libtdb.so.1\n"
+                     # "0x3bc3000000+0x209000 8ef56f789fd914e8d0678eb0cdfda1bfebb00b40@0x3bc30001a0 /usr/lib64/libltdl.so.7 - libltdl.so.7\n"
+                     # "0x7f4231b64000+0x22b000 3ca5b83798349f78b362b1ea51c8a4bc8114b8b1@0x7f4231b641a0 /usr/lib64/gio/modules/libgvfsdbus.so - libgvfsdbus.so\n"
+                     # "0x7f423192a000+0x218000 ad024a01ad132737a8cfc7c95beb7c77733a652d@0x7f423192a1a0 /usr/lib64/libgvfscommon.so.0 - libgvfscommon.so.0\n"
+                     # "0x7f423192a000+0x218000 ad024a01ad132737a8cfc7c95beb7c77733a652d@0x7f423192a1a0 /usr/lib64/libgvfscommon.so.0 - libgvfscommon.so.0\n"
+                     # "0x3bb8e00000+0x20e000 d240ac5755184a95c783bb98a2d05530e0cf958a@0x3bb8e001a0 /lib64/libudev.so.0 - libudev.so.0")
+    #
+    #print eu_unstrip_OUT
+    # we failed to get build ids from the core -> die
+    if not eu_unstrip_OUT:
+        log1("can't get build ids from the core")
+        return RETURN_FAILURE
+
+    lines = eu_unstrip_OUT.split('\n')
+    # using set ensures the unique values
+    build_ids = set()
+    libraries = set()
+    build_ids = set()
+
+    for line in lines:
+        b_ids_line = line.split()
+        if len(b_ids_line) > 2:
+            # [exe] -> the executable itself
+            # linux-vdso.so.1 -> Virtual Dynamic Shared Object
+            if b_ids_line[EXECUTABLE] not in ["linux-vdso.so.1"]:
+                build_id = b_ids_line[BUILD_ID].split('@')[0]
+                build_ids.add(build_id)
+                library = b_ids_line[LIBRARY]
+                libraries.add(library)
+                build_ids.add(build_id)
+            else:
+                log2("skipping line %s" % line)
+    log1("Found %i build_ids" % len(build_ids))
+    log1("Found %i libs" % len(libraries))
+    return build_ids
+
+def build_ids_to_path(build_ids):
+    """
+    build_id1=${build_id:0:2}
+    build_id2=${build_id:2}
+    file="usr/lib/debug/.build-id/$build_id1/$build_id2.debug"
+	"""
+    return ["/usr/lib/debug/.build-id/%s/%s.debug" % (b_id[:2], b_id[2:]) for b_id in build_ids]
+
+# beware this finds only missing libraries, but not the executable itself ..
+
+def filter_installed_debuginfos(build_ids, cache_dir):
+    # 1st pass -> search in /usr/lib
+    missing_di = []
+    files = build_ids_to_path(build_ids)
+    for debuginfo_path in files:
+        cache_debuginfo_path = cache_dir + debuginfo_path
+        log2("checking path: %s" % debuginfo_path)
+        if os.path.exists(debuginfo_path):
+            log2("found: %s" % debuginfo_path)
+            continue
+        if os.path.exists(cache_debuginfo_path):
+            log2("found: %s" % cache_debuginfo_path)
+            continue
+        log2("not found: %s" % (cache_debuginfo_path))
+        missing_di.append(debuginfo_path)
+    return missing_di
+
+tmpdir = None
+def clean_up():
+    try:
+        shutil.rmtree(tmpdir)
+    except OSError, ex:
+        print (_("Can't remove %(tmpdir_path)s: %(reason)s")
+                % {"tmpdir_path":tmpdir, "reason": ex }
+                 )
+
+def sigterm_handler(signum, frame):
+    clean_up()
+    exit(RETURN_OK)
+
+def sigint_handler(signum, frame):
+    clean_up()
+    print "\n", _("Exiting on user Command")
+    exit(RETURN_OK)
+
+import signal
+if __name__ == "__main__":
+    # abrt-server can send SIGTERM to abort the download
+    signal.signal(signal.SIGTERM, sigterm_handler)
+    # ctrl-c
+    signal.signal(signal.SIGINT, sigint_handler)
+    core = None
+    cachedir = None
+    tmpdir = None
+    keeprpms = False
+    result = RETURN_OK
+    noninteractive = False
+
+    # localization
+    init_gettext()
+
+    help_text = _("Usage: %s --core=<COREFILE> "
+                            "--tmpdir=<TMPDIR> "
+                            "--cachedir=<CACHEDIR>") % sys.argv[0]
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "vyhc:", ["help", "core=",
+                                                          "cache=", "tmpdir=",
+                                                          "keeprpms"])
+    except getopt.GetoptError, err:
+        print str(err) # prints something like "option -a not recognized"
+        sys.exit(RETURN_FAILURE)
+
+    for opt, arg in opts:
+        if opt == "-v":
+            verbose += 1
+        elif opt == "-y":
+            noninteractive = True
+        elif opt in ("--core","-c"):
+            core = arg
+        elif opt in ("--cache"):
+            cachedir = arg
+        elif opt in ("--tmpdir"):
+            tmpdir = arg
+        elif opt in ("--keeprpms"):
+            keeprpms = True
+        elif opt in ("-h", "--help"):
+            print help_text
+            sys.exit()
+
+    if not core:
+        print _("You have to specify the path to coredump.")
+        print help_text
+        exit(RETURN_FAILURE)
+    if not cachedir:
+        print _("You have to specify the path to cachedir.")
+        print help_text
+        exit(RETURN_FAILURE)
+    if not tmpdir:
+        print _("You have to specify the path to tmpdir.")
+        print help_text
+        exit(RETURN_FAILURE)
+
+    b_ids = extract_info_from_core(core)
+    if b_ids == RETURN_FAILURE:
+        exit(RETURN_FAILURE)
+    missing = filter_installed_debuginfos(b_ids, cachedir)
+    if missing:
+        log2(missing)
+        downloader = DebugInfoDownload(cache=cachedir, tmp=tmpdir)
+        result = downloader.download(missing)
+    else:
+        print _("All debuginfo seems to be available")
+        exit(RETURN_OK)
+
+    missing = filter_installed_debuginfos(b_ids, cachedir)
+    for bid in missing:
+        log1("MISSING:%s" % bid)
+
+    print _("Complete!")
+    exit(result)
+