diff options
Diffstat (limited to 'yum-presto')
| -rw-r--r-- | yum-presto/ChangeLog | 5 | ||||
| -rw-r--r-- | yum-presto/Makefile | 9 | ||||
| -rw-r--r-- | yum-presto/presto.conf | 9 | ||||
| -rw-r--r-- | yum-presto/presto.py | 464 | ||||
| -rw-r--r-- | yum-presto/shared/deltarpm.py | 86 | ||||
| -rw-r--r-- | yum-presto/shared/prestoDownload.py | 188 | ||||
| -rw-r--r-- | yum-presto/shared/prestoLog.py | 92 | ||||
| -rw-r--r-- | yum-presto/shared/prestoRepo.py | 615 | ||||
| -rw-r--r-- | yum-presto/shared/prestoThread.py | 53 | ||||
| -rw-r--r-- | yum-presto/shared/prestoTransaction.py | 140 | ||||
| -rw-r--r-- | yum-presto/shared/prestomdparser.py | 169 |
11 files changed, 355 insertions, 1475 deletions
diff --git a/yum-presto/ChangeLog b/yum-presto/ChangeLog index 05c5c41..0007bf7 100644 --- a/yum-presto/ChangeLog +++ b/yum-presto/ChangeLog @@ -1,3 +1,8 @@ +* Wed Jul 11 2007 Jonathan Dieter <jdieter@gmail.com> - 0.4.0 + - Complete rewrite by Jeremy Katz. + - Many old options and features removed in preparation for use as part of + Fedora 8. + * Tue May 1 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.10 - Use new -a option to deltarpm to only check against a certain architecture. This allows us to work completely correctly on x86_64. diff --git a/yum-presto/Makefile b/yum-presto/Makefile index 8826b30..c756c05 100644 --- a/yum-presto/Makefile +++ b/yum-presto/Makefile @@ -1,17 +1,8 @@ clean: rm -f *.pyc *.pyo *~ - cd shared; rm -f *.pyc *.pyo *~ install: mkdir -p $(DESTDIR)/usr/lib/yum-plugins install -m 644 presto.py $(DESTDIR)/usr/lib/yum-plugins mkdir -p $(DESTDIR)/etc/yum/pluginconf.d install -m 644 presto.conf $(DESTDIR)/etc/yum/pluginconf.d - mkdir -p $(DESTDIR)/usr/share/presto - install -m 644 shared/prestoRepo.py $(DESTDIR)/usr/share/presto - install -m 644 shared/prestomdparser.py $(DESTDIR)/usr/share/presto - install -m 644 shared/prestoTransaction.py $(DESTDIR)/usr/share/presto - install -m 644 shared/prestoThread.py $(DESTDIR)/usr/share/presto - install -m 644 shared/prestoLog.py $(DESTDIR)/usr/share/presto - install -m 644 shared/prestoDownload.py $(DESTDIR)/usr/share/presto - install -m 644 shared/deltarpm.py $(DESTDIR)/usr/share/presto diff --git a/yum-presto/presto.conf b/yum-presto/presto.conf index 488f69f..b7dc4a9 100644 --- a/yum-presto/presto.conf +++ b/yum-presto/presto.conf @@ -5,12 +5,3 @@ enabled=1 neverkeepdeltas=1 -# Setup for test server, enable if you are testing -#[updates] -#deltaurl=http://www.lesbg.com/jdieter/updates/fc6/i386 -# -#[extras] -#deltaurl=http://www.lesbg.com/jdieter/extras/fc6/i386 -# -#[development] -#deltaurl=http://www.lesbg.com/jdieter/updates/development/i386 diff --git a/yum-presto/presto.py b/yum-presto/presto.py index 48db24d..255372c 100644 --- a/yum-presto/presto.py +++ b/yum-presto/presto.py @@ -18,36 +18,342 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2005 Duke University # Copyright 2007 Jonathan Dieter - -from yum.plugins import TYPE_INTERACTIVE, PluginYumExit -from yum import config +# Copyright 2007 Red Hat, Inc. -- Jeremy Katz <katzj@redhat.com> import os import sys +import subprocess +import gzip +import thread +import threading +import Queue +try: + from cElementTree import iterparse +except: + from xml.etree.cElementTree import iterparse + +from yum.plugins import TYPE_CORE, PluginYumExit +import yum.Errors +import yum.misc +from urlgrabber.grabber import URLGrabError -sys.path.append("/usr/share/presto") -import deltarpm -from prestoRepo import PrestoRepository -from prestomdparser import PrestoMDParser -import prestoTransaction -import prestoLog -import prestoDownload requires_api_version = '2.1' -LOG_FILE = "/var/log/presto.log" -plugin_type = (TYPE_INTERACTIVE,) +plugin_type = (TYPE_CORE,) + +# mapping of repo.id -> PrestoInfo +pinfo = {} + +def verifyDelta(sequence, arch): + if subprocess.call(["/usr/bin/applydeltarpm", "-a", arch, + "-C", "-s", sequence]): + return False + return True + +def applyDelta(deltarpmfile, newrpmfile, arch): + if subprocess.call(["/usr/bin/applydeltarpm", "-a", arch, + deltarpmfile, newrpmfile]): + return False + return True + +def reconstruct(conduit, rpmlocal, rpmarch, deltalocal): + retlist = "" + + if not applyDelta(deltalocal, rpmlocal, rpmarch): + retlist += "Error rebuilding rpm from %s! Will download full package.\n" % os.path.basename(deltalocal) + try: + os.unlink(rpmlocal) + except: + pass + else: + # Check to see whether or not we should keep the drpms + # FIXME: Is there any way to see whether or not a Boolean option was not set? + if conduit.confBool('main', 'neverkeepdeltas'): + delete = True + elif conduit.confBool('main', 'keepdeltas'): + delete = False + elif conduit.getConf().keepcache != 0: + delete = False + else: + delete = True + + if delete: + try: + os.unlink(deltalocal) + except: + pass + return retlist + +class ReconstructionThread(threading.Thread): + def __init__(self, queue, lock, run_function): + threading.Thread.__init__(self) + self.run_function = run_function + self.queue = queue + self.lock = lock + self.can_exit = False + self.messages = "" + + def run(self): + while True: + try: + retval = self.queue.get(not self.can_exit) + except Queue.Empty: + # If we're done with our drpms and no more are coming, let's + # blow this joint + break + if retval != None: + messages = apply(self.run_function, retval) + if self.can_exit: + # If there are not going to be any more new drpms, + # send messages directly to conduit + conduit = retval[0] + if self.messages != "": + conduit.info(2, self.messages[:-1]) + self.messages = "" + if messages != "": + conduit.info(2, messages[:-1]) + else: + # We may be downloading drpms still, so queue messages + self.lock.acquire() + self.messages += messages + self.lock.release() + + +def getDelta(po, presto, rpmdb): + """Does the package have a reasonable delta for us to use?""" + + # local packages don't make sense to use a delta for... + if hasattr(po, 'pkgtype') and po.pkgtype == 'local': + return None + if po.remote_url.startswith("file:/"): + # kind of a hack, but file:/ repos are basically local + return None + + # if there's not presto info for the repo, we don't have a delta for + # the package + if not presto.has_key(po.repo.id): + return None + deltainfo = presto[po.repo.id] + + # any deltas for the new package in the repo? + nevra = "%s-%s:%s-%s.%s" %(po.name, po.epoch, po.version, + po.release, po.arch) + if not deltainfo.has_key(nevra): + return None + deltas = deltainfo[nevra] + + # check to see if we've already got the full package + local = po.localPkg() + if os.path.exists(local): + cursize = os.stat(local)[6] + totsize = long(po.size) + if po.verifyLocalPkg(): # we've got it. + return None + if cursize < totsize: # we have part of the file; do a reget + return None + os.unlink(local) + + # did we have a previous package of the same arch installed? + installed = rpmdb.searchNevra(po.name, None, None, None, po.arch) + if len(installed) == 0: + return None + + # now, let's see if there's a delta for us... + bestdelta = None + + for oldpo in installed: + evr = "%s:%s-%s" %(oldpo.epoch, oldpo.version, oldpo.release) + if not deltas.has_key(evr): + continue + delta = deltas[evr] + + # we just want to use the smallest delta + if bestdelta and delta['size'] >= bestdelta['size']: + continue + + if not verifyDelta(delta['sequence'], po.arch): + continue + + bestdelta = delta + + return bestdelta + + +def downloadPkgs(conduit, presto): + """download list of package objects handed to you, return errors""" + + errors = {} + def adderror(po, msg): + errors.setdefault(po, []).append(msg) + + # Set up thread for applying drpms + queue = Queue.Queue(0) + lock = thread.allocate_lock() + curthread = ReconstructionThread(queue, lock, reconstruct) + curthread.start() + + remote_pkgs = [] + + # see which deltas we need to download; if the delta is already + # downloaded, we can start it reconstructing in the background + for po in conduit.getDownloadPackages(): + delta = getDelta(po, presto, conduit.getRpmDB()) + if delta is None: + continue + + # verify the delta if it already exists + deltadir = os.path.join(po.repo.cachedir, 'deltas') + if not os.path.isdir(deltadir): + try: + os.mkdir(deltadir) + except OSError: + continue + deltapath = os.path.join(deltadir, + os.path.basename(delta['filename'])) + if os.path.exists(deltapath): + try: + conduit._base.verifyChecksum(deltapath, delta['checksum_type'], + delta['checksum']) + except URLGrabError, e: + if po.repo.cache: + raise Errors.RepoError, "Caching enabled and local cache for %s doesn't match checksum" %(deltapath,) + else: + cursize = os.stat(deltapath)[6] + totsize = long(delta['size']) + if cursize >= totsize: + os.unlink(deltapath) + + remote_packages.append( (po, delta) ) + else: + # Deltarpm is local and good, put it in the rebuild thread. + conduit.info(5, "using local copy of deltarpm for %s" % po) + queue.put((conduit, po.localpath, po.arch, deltapath)) + continue + else: + remote_pkgs.append( (po, delta) ) + + # now we need to do downloads + i = 0 + for (po, delta) in remote_pkgs: + i += 1 + # FIXME: verifyChecksum should handle the urlgrabber objects... + checkfunc = (lambda fo, csumtype, csum: + conduit._base.verifyChecksum(fo.filename, csumtype, csum), + (delta['checksum_type'], + delta['checksum']), {}) + + deltadir = os.path.join(po.repo.cachedir, 'deltas') + deltapath = os.path.join(deltadir, + os.path.basename(delta['filename'])) + + # FIXME: this should be moved into _getFile + dirstat = os.statvfs(deltadir) + if (dirstat.f_bavail * dirstat.f_bsize) <= long(po.size): + adderror(po, 'Insufficient space in download directory %s ' + 'to download' % (deltadir,)) + continue + try: + text = "(%s/%s): %s" %(i, len(remote_pkgs), + os.path.basename(delta['filename'])) + deltafile = po.repo._getFile(url=po.basepath, + relative=delta['filename'], + local=deltapath, + checkfunc=checkfunc, + text=text, + cache=po.repo.cache) + except Errors.RepoError, e: + adderror(po, str(e)) + else: + queue.put((conduit, po.localpath, po.arch, deltafile)) + + if errors.has_key(po): + del errors[po] + + # Check for waiting messages from building thread + lock.acquire() + if curthread.messages != "": + conduit.info(2, curthread.messages[:-1]) + curthread.messages = "" + lock.release() + + conduit.info(2, "Rebuilding rpms from deltarpms") + + # Tell build thread that there are no more drpms and wait for it to exit + curthread.can_exit = True + queue.put(None) + curthread.join() + + if curthread.messages != "": + conduit.info(2, curthread.messages[:-1]) + curthread.messages = "" + + return errors + +class DeltaInfo(object): + def __init__(self, elem): + self.epoch = elem.get("oldepoch") + self.version = elem.get("oldversion") + self.release = elem.get("oldrelease") + + self.filename = self.sequence = self.size = self.checksum = self.checksum_type = None + + for x in elem.getchildren(): + if x.tag == "checksum": + self.checksum_type = x.get("type") + setattr(self, x.tag, x.text) + + def evr(self): + return "%s:%s-%s" %(self.epoch, self.version, self.release) + + def __str__(self): + return "filename: %s, sequence: %s, size: %s, checksum (%s) = %s" % (self.filename, self.sequence, self.size, self.checksum_type, self.checksum) + + def __getitem__(self, key): + return getattr(self, key) + +class NewPackage(object): + def __init__(self, elem): + for prop in ("name", "version", "release", "epoch", "arch"): + setattr(self, prop, elem.get(prop)) + + self.deltas = {} + for child in elem.getchildren(): + if child.tag != "delta": + continue + d = DeltaInfo(child) + self.deltas[d.evr()] = d + + def nevra(self): + return "%s-%s:%s-%s.%s" %(self.name, self.epoch, self.version, + self.release, self.arch) + + def __str__(self): + return "%s <== %s" % (self.nevra(), self.deltas) + + def has_key(self, key): + return self.deltas.has_key(key) + def __getitem__(self, key): + return self.deltas[key] + +class PrestoParser(object): + def __init__(self, filename): + self.deltainfo = {} + + if filename.endswith(".gz"): + fo = gzip.open(filename) + else: + fo = open(filename, 'rt') + for event, elem in iterparse(fo): + if elem.tag == "newpackage": + p = NewPackage(elem) + self.deltainfo[p.nevra()] = p + + def getDeltas(self): + return self.deltainfo -rpm_size = 0 -drpm_size = 0 -drpm_count = 0 -log = None # Configuration stuff def config_hook(conduit): - # Set up repository specific deltarpm url and mirrorlist - config.RepoConf.deltaurl = config.UrlListOption() - config.RepoConf.deltamirrorlist = config.UrlOption() - # Add --disable-presto option parser = conduit.getOptParser() parser.add_option('', '--disablepresto', dest='disablepresto', @@ -58,106 +364,36 @@ def config_hook(conduit): def postreposetup_hook(conduit): opts, commands = conduit.getCmdLine() if not opts.disablepresto: - conduit.info(2, 'Setting up Presto') + conduit.info(2, 'Setting up and reading Presto delta metadata') for active_repo in conduit.getRepos().listEnabled(): - p_repo = PrestoRepository(active_repo, conduit) - p_repo.setup(conduit.getConf().cache) - - conduit.info(2, 'Reading Presto metadata in from local files') - for active_repo in conduit.getRepos().listEnabled(): - xml = active_repo.p_repo.getPrestoXML() - if active_repo.p_repo.enabled: - xmldata = active_repo.p_repo.repoXML.getData('deltas') - (ctype, csum) = xmldata.checksum - parser = PrestoMDParser(xml) - active_repo.p_repo.deltalist = parser.getDeltaList() + try: + deltamd = active_repo.retrieveMD("prestodelta") + except yum.Errors.RepoMDError: + conduit.info(2, "No Presto metadata available for %s" %(active_repo,)) + continue + pinfo[active_repo.id] = PrestoParser(deltamd).getDeltas() else: conduit.info(5, '--disablepresto specified - Presto disabled') - -def postresolve_hook(conduit): - global rpm_size - global drpm_size - global drpm_count - + +def predownload_hook(conduit): opts, commands = conduit.getCmdLine() - if not opts.disablepresto: - # Cycle through packages to see if there's a deltarpm available - for newpkg in conduit.getTsInfo(): - if newpkg.ts_state != "e": - (chosen_drpm, installed, local, drpm_enabled) = prestoTransaction.find_available_drpms(conduit, newpkg) - - # If a drpm was found, change certain package information so it reflects - # the drpm, not the rpm. - if chosen_drpm != None: - newpkg.po.has_drpm = True - conduit.info(2, "Found deltarpm update for %s.%s %s:%s-%s" % (newpkg.name, newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release)) - # In yum 3.0.x, this doesn't get defined if you run "yum update x" rather than "yum update" - rpm_size += int(newpkg.po.size) - drpm_size += int(chosen_drpm['size']) - newpkg.po.realpackagesize = newpkg.po.size - if hasattr(newpkg.po, 'packagesize'): - newpkg.po.packagesize = chosen_drpm['size'] - else: - newpkg.po.simple['packagesize'] = chosen_drpm['size'] - newpkg.po.deltasize = chosen_drpm['size'] - newpkg.po.deltarelativepath = chosen_drpm['drpm_filename'] - newpkg.po.deltachecksumtype = chosen_drpm['checksum_type'] - newpkg.po.deltachecksum = chosen_drpm['checksum'] - newpkg.po.deltalocalpath = newpkg.po.repo.deltasdir + "/" + os.path.basename(chosen_drpm['drpm_filename']) - newpkg.po.to = newpkg - newpkg.po.hasdrpm = True - newpkg.repoid = newpkg.po.repo.id + " *" - drpm_count += 1 - else: - if installed and drpm_enabled and not local: - try: - rpm_size += int(newpkg.po.size) - drpm_size += int(newpkg.po.size) - except: - pass + if opts.disablepresto or len(conduit.getDownloadPackages()) == 0: return - -def predownload_hook(conduit): - global drpm_count - global log - - # Set up logging - log = prestoLog.PrestoLog(conduit, LOG_FILE) + conduit.info(2, "Downloading DeltaRPMs:") - pkglist = conduit.getDownloadPackages() - - opts, commands = conduit.getCmdLine() - if not opts.disablepresto and drpm_count > 0: - conduit.info(2, "Downloading DeltaRPMs:") - - # Download deltarpms - problems = prestoDownload.downloadPkgs(conduit, pkglist, log) - - # If 'exitondownloaderror' is on, exit - if conduit.confBool('main', 'exitondownloaderror') and len(problems.keys()) > 0: - errstring = 'Error Downloading Packages:\n' - for key in problems.keys(): - errors = misc.unique(problems[key]) - for error in errors: - errstring += ' %s: %s\n' % (key, error) - raise PluginYumExit(errstring) - - else: - conduit.info(2, "Downloading RPMs:") + # Download deltarpms + problems = downloadPkgs(conduit, pinfo) -def posttrans_hook(conduit): - global rpm_size - global drpm_size - global log - - log.close() - - if rpm_size > 0: - drpm_string = prestoTransaction.format_number(drpm_size) - rpm_string = prestoTransaction.format_number(rpm_size) - - conduit.info(2, "Size of all updates downloaded from Presto-enabled repositories: %s" % drpm_string) - conduit.info(2, "Size of updates that would have been downloaded if Presto wasn't enabled: %s" % rpm_string) - conduit.info(2, "This is a savings of %i percent" % (100 - ((drpm_size * 100) / rpm_size))) + # If 'exitondownloaderror' is on, exit + if conduit.confBool('main', 'exitondownloaderror') and \ + len(problems.keys()) > 0: + errstring = 'Error Downloading DeltaRPMs:\n' + for key in problems.keys(): + errors = yum.misc.unique(problems[key]) + for error in errors: + errstring += ' %s: %s\n' % (key, error) + raise PluginYumExit(errstring) + +# FIXME: would be good to give an idea to people of what they saved diff --git a/yum-presto/shared/deltarpm.py b/yum-presto/shared/deltarpm.py deleted file mode 100644 index 97a3cc6..0000000 --- a/yum-presto/shared/deltarpm.py +++ /dev/null @@ -1,86 +0,0 @@ -# author: Jonathan Dieter <jdieter@gmail.com> -# -# mostly taken from deltarpm.py created by -# Lars Herrmann <herrmann@redhat.com> -# and modified for Presto by -# Ahmed Kamal <email.ahmedkamal@googlemail.com> -# -# license: GPL (see COPYING file in distribution) -# -# this module provides a python wrapper around deltarpm tools written by suse -# -# TODO: catch exceptions wherever possible and raise useful ones ;) -# see TODO lines in methods - -APPLY='/usr/bin/applydeltarpm' - -import popen2 -import string -import os - -class Process: - """wrapper class to execute programs and return exitcode and output (stdout and stderr combined)""" - def __init__(self, conduit): - self.__stdout=None - self.__returncode=None - self.__command=None - self.__args=None - self.conduit = conduit - - def run(self, command, *args): - self.__command=command - self.__args=args - cmdline=command+" "+string.join(args, " ") - self.conduit.info(7, '%s.%s: executing %s' % (self.__class__, 'run', cmdline)) - pipe = popen2.Popen4(cmdline) - self.__stdout=pipe.fromchild.read() - retcode = pipe.wait() - if os.WIFEXITED(retcode): - self.__returncode = os.WEXITSTATUS(retcode) - else: - self.__returncode = retcode - # fallback to old implementation - works better ? - #stdoutp = os.popen(cmdline,'r',1) - #self.__stdout = stdoutp.read() - #retcode = stdoutp.close() - #if retcode is None: - # self.__returncode = 0 - #else: - # self.__returncode = retcode - - def getOutput(self): - return self.__stdout - - def returnCode(self): - return self.__returncode - -class DeltaRpmWrapper: - """wrapper around deltarpm binaries - implement methods for applying and verifying delta rpms - - raises exceptions if exitcode of binaries was != 0""" - - def __init__(self, conduit): - self.conduit = conduit - self.conduit.info(7, '%s.%s: created' % (self.__class__, '__init__')) - - def apply(self, arch, newrpmfile, deltarpmfile): - """wraps execution of applydeltarpm [-r oldrpm] deltarpm newrpm - - constructs file names and paths based on given RpmDescription and instance settings for directories""" - # TODO: test args for type == instance and __class__ == RpmDescription - self.conduit.info(7, '%s.apply(%s,%s)' % (self.__class__, newrpmfile, deltarpmfile)) - p=Process(self.conduit) - # targetrpm filename - p.run(APPLY, '-a', arch, deltarpmfile, newrpmfile) - if p.returnCode(): - # in case of error, raise exception - raise Exception("Could not apply deltarpm: %d" % (p.returnCode())) - return newrpmfile - - def verifySequence(self, arch, sequence): - """wraps execution of applydeltarpm [-r oldrpm] -s seqfilecontent - - constructs file names and paths based on given RpmDescription and instance settings for directories""" - self.conduit.info(7, '%s.verify(%s)' % (self.__class__, sequence)) - p = Process(self.conduit) - p.run(APPLY, '-a', arch, '-C', '-s', sequence) - if p.returnCode(): - # in case of error, raise exception - raise Exception("Could not verify sequence of deltarpm: %d" % (p.returnCode())) diff --git a/yum-presto/shared/prestoDownload.py b/yum-presto/shared/prestoDownload.py deleted file mode 100644 index 1c29608..0000000 --- a/yum-presto/shared/prestoDownload.py +++ /dev/null @@ -1,188 +0,0 @@ -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# -# Copyright 2005 Duke University -# Copyright 2007 Jonathan Dieter - -import os -from yum import misc -from yum import Errors -from yum import types -from urlgrabber.grabber import URLGrabError -import deltarpm -import prestoThread -import Queue -import thread - -def verifyDelta(fo, po, conduit, raiseError): - """verifies the deltarpm is what we expect it to be - raiseError = defaults to 0 - if 1 then will raise - a URLGrabError if the file does not check out. - otherwise it returns false for a failure, true for success""" - - if type(fo) is types.InstanceType: - fo = fo.filename - - try: - verifyChecksum(fo, po.deltachecksumtype, po.deltachecksum) - except: - if raiseError: - raise URLGrabError(-1, 'Package does not match intended download') - else: - return False - - return True - - -def verifyChecksum(filename, checksumType, csum): - """Verify the checksum of the file versus the - provided checksum""" - - try: - - filesum = misc.checksum(checksumType, filename) - except Errors.MiscError, e: - raise URLGrabError(-3, 'Could not perform checksum') - - if filesum != csum: - raise URLGrabError(-1, 'Package does not match checksum') - - return 0 - -def reconstruct(conduit, po, log): - deltalocal = po.deltalocalpath - retlist = "" - - drpm = deltarpm.DeltaRpmWrapper(conduit) - - try: - drpm.apply(po.arch, po.localpath, deltalocal) - except: - retlist += "Error rebuilding rpm from %s! Will download full package.\n" % os.path.basename(deltalocal) - try: - os.unlink(po.localpath) - except: - pass - else: - # log success - log.log(po.oldpkg_string, po.newpkg_string, os.stat(po.localpath)[6], os.stat(deltalocal)[6]) - - #retlist += "Built %s from deltarpm\n" % os.path.basename(po.localpath) - # Check to see whether or not we should keep the drpms - # FIXME: Is there any way to see whether or not a Boolean option was not set? - if conduit.confBool('main', 'neverkeepdeltas'): - delete = True - elif conduit.confBool('main', 'keepdeltas'): - delete = False - elif conduit.getConf().keepcache != 0: - delete = False - else: - delete = True - - if delete: - try: - os.unlink(deltalocal) - except: - pass - return retlist - - - -def downloadPkgs(conduit, pkglist, log): - """download list of package objects handed to you, return errors""" - - opts, commands = conduit.getCmdLine() - - errors = {} - def adderror(po, msg): - errors.setdefault(po, []).append(msg) - - # Set up thread for applying drpms - queue = Queue.Queue(0) - lock = thread.allocate_lock() - curthread = prestoThread.ReconstructionThread(queue, lock, reconstruct) - curthread.start() - - # Check whether drpm is already downloaded - repo_cached = False - remote_pkgs = [] - for po in conduit.getDownloadPackages(): - if hasattr(po, 'has_drpm') and po.has_drpm: - local = po.deltalocalpath - if os.path.exists(local): - cursize = os.stat(local)[6] - totsize = long(po.deltasize) - try: - verifyChecksum(local, po.deltachecksumtype, po.deltachecksum) - except: - if po.repo.p_repo.cache: - repo_cached = True - adderror(po, 'package fails checksum but caching is ' - 'enabled for %s' % po.repo.p_repo.id) - - if cursize >= totsize: # otherwise keep it around for regetting - os.unlink(local) - else: - # Deltarpm is local and good, let's put it in the rebuild thread. - conduit.info(5, "using local copy of deltarpm for %s" % po) - queue.put((conduit, po, log)) - continue - remote_pkgs.append(po) - - # Download deltarpms and send them to another thread to be rebuilt - i = 0 - for po in remote_pkgs: - i += 1 - checkfunc = (verifyDelta, (po, conduit, 1), {}) - cache = po.repo.p_repo.http_caching != 'none' - dirstat = os.statvfs(po.repo.deltasdir) - if (dirstat.f_bavail * dirstat.f_bsize) <= long(po.size): - adderror(po, 'Insufficient space in download directory %s ' - 'to download' % po.repo.deltasdir) - continue - try: - text = '(%s/%s): %s' % (i, len(remote_pkgs), os.path.basename(po.deltarelativepath)) - deltalocal = po.repo.p_repo.getPackage(po, checkfunc=checkfunc, text=text, cache=cache) - except Errors.RepoError, e: - adderror(po, str(e)) - else: - queue.put((conduit, po, log)) - - if errors.has_key(po): - del errors[po] - - if hasattr(po, 'realpackagesize'): - po.packagesize = po.realpackagesize - del po.realpackagesize - - # Check for waiting messages from building thread - lock.acquire() - if curthread.messages != "": - conduit.info(2, curthread.messages[:-1]) - curthread.messages = "" - lock.release() - - conduit.info(2, "Rebuilding rpms from deltarpms") - - # Tell build thread that there are no more drpms and wait for it to exit - curthread.can_exit = True - queue.put(None) - curthread.join() - - if curthread.messages != "": - conduit.info(2, curthread.messages[:-1]) - curthread.messages = "" - - return errors - diff --git a/yum-presto/shared/prestoLog.py b/yum-presto/shared/prestoLog.py deleted file mode 100644 index 1323346..0000000 --- a/yum-presto/shared/prestoLog.py +++ /dev/null @@ -1,92 +0,0 @@ -# author: Jonathan Dieter <jdieter@gmail.com> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# Copyright 2005 Duke University - -class PrestoLog: - def __init__(self, conduit, log_filename): - # Open log file for reading - try: - log_file = open(log_filename, "r") - log_exists = True - except: - conduit.info(7, "Info: %s doesn't exist. Will create." % log_filename) - log_exists = False - - # Log file doesn't exist, create - if not log_exists: - try: - log_file = open(log_filename, "w") - log_file.close() - log_exists = True - except: - conduit.info(2, "Warning: Unable to write to %s" % log_filename) - if log_exists: - try: - log_file = open(log_filename, "r") - except: - conduit.info(2, "Warning: Unable to open %s for reading." % log_filename) - log_exists = False - - # Cycle through items already in log so we can come up with total savings - if log_exists: - self.total_rpm_size = 0 - self.total_drpm_size = 0 - - # Get rid of header line - log_file.readline() - - data = log_file.readline() - while data != "": - fc = data.rfind("-") - sc = data.rfind("-", 0, fc-1) - tc = data.rfind("-", 0, sc-1) - lc = data.rfind("-", 0, tc-1) - if lc != -1 and tc != -1 and sc != -1 and fc != -1: - self.total_rpm_size += int(data[lc+1:tc]) - self.total_drpm_size += int(data[tc+1:sc]) - data = log_file.readline() - log_file.close() - - try: - log_file = open(log_filename, "a") - except: - conduit.info(2, "Warning: Unable to open %s for writing." % log_filename) - self.log_file = None - else: - self.log_file = log_filename - log_file.close() - - def log(self, oldrpm_name, newrpm_name, rpm_size, drpm_size): - # Write data to log - self.total_rpm_size += rpm_size - self.total_drpm_size += drpm_size - if self.log_file != None: - try: - log_file = open(self.log_file, "a") - except: - pass - else: - log_file.write("%s => %s - %i - %i - %i - %i\n" % (oldrpm_name, newrpm_name, rpm_size, drpm_size, 100 - ((drpm_size * 100) / rpm_size), 100 - ((self.total_drpm_size * 100) / self.total_rpm_size))) - log_file.close() - - - def close(self): - if self.log_file != None: - try: - self.log_file.close() - except: - pass - diff --git a/yum-presto/shared/prestoRepo.py b/yum-presto/shared/prestoRepo.py deleted file mode 100644 index bc1b188..0000000 --- a/yum-presto/shared/prestoRepo.py +++ /dev/null @@ -1,615 +0,0 @@ -# author: Jonathan Dieter <jdieter@gmail.com> -# -# mostly taken from yumRepo.py (part of yum) with a few minor modifications -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# Copyright 2005 Duke University - -import os -import re -import time -import types -import urlparse - -from yum import Errors -from urlgrabber.grabber import URLGrabber -import urlgrabber.mirror -from urlgrabber.grabber import URLGrabError -from yum.repos import Repository -from yum import repoMDObject -from yum import parser -from yum import config -from yum import misc - -class PrestoRepository(Repository): - """ - This is an actual repository object - - Configuration attributes are pulled in from config.RepoConf. - """ - - def __init__(self, repo, conduit): - Repository.__init__(self, repo.id) - - # If there's a specific deltarpm url, use that - is_different = False - if conduit.confString(repo.id, 'deltaurl'): - self.baseurl = [conduit.confString(repo.id, 'deltaurl')] - is_different = True - conduit.info(5, 'Manual url set from presto.conf: %s' % self.baseurl) - elif repo.deltaurl != [] and repo.deltaurl != None: - self.baseurl = repo.deltaurl - is_different = True - conduit.info(5, 'Manual url set from repository conf file: %s' % self.baseurl) - else: - self.baseurl = repo.baseurl - - # If there's a specific mirrorlist, use that - if conduit.confString(repo.id, 'deltamirrorlist'): - self.mirrorlist = conduit.confString(repo.id, 'deltamirrorlist') - self.baseurl = None - is_different = True - conduit.info(5, 'Manual mirrorlist set from presto.conf: %s' % self.mirrorlist) - elif repo.deltamirrorlist != None: - self.mirrorlist = repo.deltamirrorlist - self.baseurl = None - is_different = True - conduit.info(5, 'Manual mirrorlist set from repository conf file: %s' % self.mirrorlist) - else: - if self.baseurl == repo.baseurl: - self.mirrorlist = repo.mirrorlist - else: - self.mirrorlist = None - - self.conduit = conduit - self.urls = [] - self.is_different = is_different - if is_different: - self.repoMDFile = 'repodata/prestomd.xml' - self.metadata_cookie_fn = 'presto_cachecookie' - else: - self.repoMDFile = 'repodata/repomd.xml' - self.metadata_cookie_fn = 'cachecookie' - self.repoXML = None - self.cache = 0 - self.mirrorlistparsed = 0 - self.yumvar = {} # empty dict of yumvariables for $string replacement - self._proxy_dict = {} - self.http_headers = {} - - # throw in some stubs for things that will be set by the config class - self.basecachedir = "" - self.cachedir = "" - self.pkgdir = "" - self.hdrdir = "" - self.enabled = True - - # holder for stuff we've grabbed - self.retrieved = { 'deltas':0 } - - # callbacks - self.keepalive = repo.keepalive - self.bandwidth = repo.bandwidth - self.retries = repo.retries - self.throttle = repo.throttle - self.proxy = repo.proxy - self.proxy_username = repo.proxy_username - self.proxy_password = repo.proxy_password - self.timeout = repo.timeout - self.http_caching = repo.http_caching - self.failovermethod = repo.failovermethod - self.metadata_expire = repo.metadata_expire - self.basecachedir = repo.basecachedir - self.callback = repo.callback - self.failure_obj = repo.failure_obj - self.mirror_failure_obj = repo.mirror_failure_obj - self.interrupt_callback = repo.interrupt_callback - self.drpm_list = {} - self.parent = repo - repo.p_repo = self - - - def __getProxyDict(self): - self.doProxyDict() - if self._proxy_dict: - return self._proxy_dict - return None - - # consistent access to how proxy information should look (and ensuring - # that it's actually determined for the repo) - proxy_dict = property(__getProxyDict) - - def ready(self): - """Returns true if this repository is setup and ready for use.""" - return self.repoXML is not None - - def __cmp__(self, other): - if self.id > other.id: - return 1 - elif self.id < other.id: - return -1 - else: - return 0 - - def __str__(self): - return self.id - - def _checksum(self, sumtype, file, CHUNK=2**16): - """takes filename, hand back Checksum of it - sumtype = md5 or sha - filename = /path/to/file - CHUNK=65536 by default""" - try: - return misc.checksum(sumtype, file, CHUNK) - except (Errors.MiscError, EnvironmentError), e: - raise Errors.RepoError, 'Error opening file for checksum: %s' % e - - def dump(self): - output = '[%s]\n' % self.id - vars = ['id', 'bandwidth', 'enabled', - 'keepalive', 'proxy', - 'proxy_password', 'proxy_username', - 'retries', 'throttle', 'timeout', 'mirrorlist', - 'cachedir' ] - vars.sort() - for attr in vars: - output = output + '%s = %s\n' % (attr, getattr(self, attr)) - output = output + 'baseurl =' - for url in self.urls: - output = output + ' %s\n' % url - - return output - - def check(self): - """self-check the repo information - if we don't have enough to move - on then raise a repo error""" - if len(self.urls) < 1: - raise Errors.RepoError, \ - 'Cannot find a valid deltaurl for repo: %s' % self.id - - def doProxyDict(self): - if self._proxy_dict: - return - - self._proxy_dict = {} # zap it - proxy_string = None - if self.proxy not in [None, '_none_']: - proxy_string = '%s' % self.proxy - if self.proxy_username is not None: - proxy_parsed = urlparse.urlsplit(self.proxy, allow_fragments=0) - proxy_proto = proxy_parsed[0] - proxy_host = proxy_parsed[1] - proxy_rest = proxy_parsed[2] + '?' + proxy_parsed[3] - proxy_string = '%s://%s@%s%s' % (proxy_proto, - self.proxy_username, proxy_host, proxy_rest) - - if self.proxy_password is not None: - proxy_string = '%s://%s:%s@%s%s' % (proxy_proto, - self.proxy_username, self.proxy_password, - proxy_host, proxy_rest) - - if proxy_string is not None: - self._proxy_dict['http'] = proxy_string - self._proxy_dict['https'] = proxy_string - self._proxy_dict['ftp'] = proxy_string - - def __headersListFromDict(self): - """Convert our dict of headers to a list of 2-tuples for urlgrabber.""" - headers = [] - - keys = self.http_headers.keys() - for key in keys: - headers.append((key, self.http_headers[key])) - - return headers - - def setupGrab(self): - """sets up the grabber functions with the already stocked in urls for - the mirror groups""" - - if self.failovermethod == 'roundrobin': - mgclass = urlgrabber.mirror.MGRandomOrder - else: - mgclass = urlgrabber.mirror.MirrorGroup - - headers = tuple(self.__headersListFromDict()) - - self.grabfunc = URLGrabber(keepalive=self.keepalive, - bandwidth=self.bandwidth, - retry=self.retries, - throttle=self.throttle, - progress_obj=self.callback, - proxies = self.proxy_dict, - failure_callback=self.failure_obj, - interrupt_callback=self.interrupt_callback, - timeout=self.timeout, - http_headers=headers, - reget='simple') - - self.grab = mgclass(self.grabfunc, self.urls, - failure_callback=self.mirror_failure_obj) - - def dirSetup(self): - """make the necessary dirs, if possible, raise on failure""" - - cachedir = os.path.join(self.parent.basecachedir, self.id) - deltasdir = os.path.join(cachedir, 'deltas') - self.parent.setAttribute('deltasdir', deltasdir) - self.setAttribute('cachedir', cachedir) - - cookie = cachedir + '/' + self.metadata_cookie_fn - self.setAttribute('metadata_cookie', cookie) - - for dir in [cachedir, self.parent.deltasdir]: - if self.cache == 0: - if os.path.exists(dir) and os.path.isdir(dir): - continue - else: - try: - os.makedirs(dir, mode=0755) - except OSError, e: - raise Errors.RepoError, \ - "Error making cache directory: %s error was: %s" % (dir, e) - else: - if not os.path.exists(dir): - raise Errors.RepoError, \ - "Cannot access repository dir %s" % dir - - def baseurlSetup(self): - """go through the baseurls and mirrorlists and populate self.urls - with valid ones, run self.check() at the end to make sure it worked""" - - goodurls = [] - if self.mirrorlist and not self.mirrorlistparsed: - mirrorurls = getMirrorList(self.mirrorlist, self.proxy_dict) - self.mirrorlistparsed = 1 - for url in mirrorurls: - url = parser.varReplace(url, self.yumvar) - self.baseurl.append(url) - - for url in self.baseurl: - url = parser.varReplace(url, self.yumvar) - (s,b,p,q,f,o) = urlparse.urlparse(url) - if s not in ['http', 'ftp', 'file', 'https']: - print 'not using ftp, http[s], or file for repos, skipping - %s' % (url) - continue - else: - goodurls.append(url) - - self.setAttribute('urls', goodurls) - self.check() - self.setupGrab() # update the grabber for the urls - - def __get(self, url=None, relative=None, local=None, start=None, end=None, - copy_local=0, checkfunc=None, text=None, reget='simple', cache=True): - """retrieve file from the mirrorgroup for the repo - relative to local, optionally get range from - start to end, also optionally retrieve from a specific baseurl""" - - # if local or relative is None: raise an exception b/c that shouldn't happen - # if url is not None - then do a grab from the complete url - not through - # the mirror, raise errors as need be - # if url is None do a grab via the mirror group/grab for the repo - # return the path to the local file - - # Turn our dict into a list of 2-tuples - headers = self.__headersListFromDict() - - # We will always prefer to send no-cache. - if not (cache or self.http_headers.has_key('Pragma')): - headers.append(('Pragma', 'no-cache')) - - headers = tuple(headers) - - if local is None or relative is None: - raise Errors.RepoError, \ - "get request for Repo %s, gave no source or dest" % self.id - - if self.cache == 1: - if os.path.exists(local): # FIXME - we should figure out a way - return local # to run the checkfunc from here - - else: # ain't there - raise - raise Errors.RepoError, \ - "Caching enabled but no local cache of %s from %s" % (local, - self) - if url is not None: - ug = URLGrabber(keepalive = self.keepalive, - bandwidth = self.bandwidth, - retry = self.retries, - throttle = self.throttle, - progress_obj = self.callback, - copy_local = copy_local, - reget = reget, - proxies = self.proxy_dict, - failure_callback = self.failure_obj, - interrupt_callback=self.interrupt_callback, - timeout=self.timeout, - checkfunc=checkfunc, - http_headers=headers, - ) - - remote = url + '/' + relative - - try: - result = ug.urlgrab(remote, local, - text=text, - range=(start, end), - ) - except URLGrabError, e: - raise Errors.RepoError, \ - "failed to retrieve %s from %s\nerror was %s" % (relative, self.id, e) - - else: - try: - result = self.grab.urlgrab(relative, local, - text = text, - range = (start, end), - copy_local=copy_local, - reget = reget, - checkfunc=checkfunc, - http_headers=headers, - ) - except URLGrabError, e: - raise Errors.RepoError, "failure: %s from %s: %s" % (relative, self.id, e) - - return result - - def getPackage(self, package, checkfunc = None, text = None, cache = True): - remote = package.deltarelativepath - local = package.deltalocalpath - if hasattr(package, 'basepath'): - basepath = package.basepath - else: - basepath = package.returnSimple('basepath') - - return self.__get(url=basepath, - relative=remote, - local=local, - checkfunc=checkfunc, - text=text, - cache=cache - ) - - def metadataCurrent(self): - """Check if there is a metadata_cookie and check its age. If the - age of the cookie is less than metadata_expire time then return true - else return False""" - - val = False - if os.path.exists(self.metadata_cookie): - cookie_info = os.stat(self.metadata_cookie) - if cookie_info[8] + self.metadata_expire > time.time(): - val = True - # WE ARE FROM THE FUTURE!!!! - elif cookie_info[8] > time.time(): - val = False - return val - - def setMetadataCookie(self): - """if possible, set touch the metadata_cookie file""" - - check = self.metadata_cookie - if not os.path.exists(self.metadata_cookie): - check = self.cachedir - - if os.access(check, os.W_OK): - fo = open(self.metadata_cookie, 'w+') - fo.close() - del fo - - - def setup(self, cache): - try: - self.cache = cache - self.baseurlSetup() - self.dirSetup() - except Errors.RepoError, e: - raise - - try: - self._loadRepoXML(text=self) - except Errors.RepoError, e: - raise Errors.RepoError, ('Cannot open/read %s file for repository: %s' % (self.repoMDFile, self)) - - - def _loadRepoXML(self, text=None): - """retrieve/check/read in repomd.xml from the repository""" - - remote = self.repoMDFile - if self.is_different: - local = self.cachedir + '/prestomd.xml' - else: - local = self.cachedir + '/repomd.xml' - - if self.repoXML is not None: - return - - if self.cache or self.metadataCurrent(): - if not os.path.exists(local): - raise Errors.RepoError, 'Cannot find %s file for %s' % (self.repoMDFile, self) - else: - result = local - else: - checkfunc = (self._checkRepoXML, (), {}) - try: - result = self.__get(relative=remote, - local=local, - copy_local=1, - text=text, - reget=None, - checkfunc=checkfunc, - cache=self.http_caching == 'all') - - - except URLGrabError, e: - raise Errors.RepoError, 'Error downloading file %s: %s' % (local, e) - # if we have a 'fresh' repomd.xml then update the cookie - self.setMetadataCookie() - - try: - self.repoXML = repoMDObject.RepoMD(self.id, result) - except Errors.RepoMDError, e: - raise Errors.RepoError, 'Error importing %s from %s: %s' % (self.repoMDFile, self, e) - - def _checkRepoXML(self, fo): - if type(fo) is types.InstanceType: - filepath = fo.filename - else: - filepath = fo - - try: - repoMDObject.RepoMD(self.id, filepath) - except Errors.RepoMDError, e: - raise URLGrabError(-1, 'Error importing %s for %s: %s' % (self.repoMDFile, self, e)) - - - def checkMD(self, fn, mdtype): - """check the metadata type against its checksum""" - - thisdata = self.repoXML.getData(mdtype) - - (r_ctype, r_csum) = thisdata.checksum # get the remote checksum - - if type(fn) == types.InstanceType: # this is an urlgrabber check - file = fn.filename - else: - file = fn - - try: - l_csum = self._checksum(r_ctype, file) # get the local checksum - except Errors.RepoError, e: - raise URLGrabError(-3, 'Error performing checksum') - - if l_csum == r_csum: - return 1 - else: - raise URLGrabError(-1, 'Metadata file does not match checksum') - - - - def retrieveMD(self, mdtype): - """base function to retrieve metadata files from the remote url - returns the path to the local metadata file of a 'mdtype' - mdtype must be 'deltas'.""" - try: - thisdata = self.repoXML.getData(mdtype) - except Errors.RepoMDError: - self.enabled = False - self.conduit.info(5, "No drpms available for %s" % self.id) - return - - (r_base, remote) = thisdata.location - fname = os.path.basename(remote) - local = self.cachedir + '/' + fname - - if self.retrieved.has_key(mdtype): - if self.retrieved[mdtype]: # got it, move along - return local - - if self.cache == 1: - if os.path.exists(local): - try: - self.checkMD(local, mdtype) - except URLGrabError, e: - raise Errors.RepoError, \ - "Caching enabled and local cache: %s does not match checksum" % local - else: - return local - - else: # ain't there - raise - raise Errors.RepoError, \ - "Caching enabled but no local cache of %s from %s" % (local, - self) - - if os.path.exists(local): - try: - self.checkMD(local, mdtype) - except URLGrabError, e: - pass - else: - self.retrieved[mdtype] = 1 - return local # it's the same return the local one - - try: - checkfunc = (self.checkMD, (mdtype,), {}) - local = self.__get(relative=remote, local=local, copy_local=1, - checkfunc=checkfunc, reget=None, - cache=self.http_caching == 'all') - except URLGrabError, e: - raise Errors.RepoError, \ - "Could not retrieve %s matching remote checksum from %s" % (local, self) - else: - self.retrieved[mdtype] = 1 - return local - - - def getPrestoXML(self): - """this gets you the path to the primary.xml file, retrieving it if we - need a new one""" - - return self.retrieveMD('deltas') - - def setCallback(self, callback): - self.callback = callback - self.setupGrab() - - def setFailureObj(self, failure_obj): - self.failure_obj = failure_obj - self.setupGrab() - - def setMirrorFailureObj(self, failure_obj): - self.mirror_failure_obj = failure_obj - self.setupGrab() - - def setInterruptCallback(self, callback): - self.interrupt_callback = callback - self.setupGrab() - -def getMirrorList(mirrorlist, pdict = None): - """retrieve an up2date-style mirrorlist file from a url, - we also s/$ARCH/$BASEARCH/ and move along - returns a list of the urls from that file""" - - returnlist = [] - if hasattr(urlgrabber.grabber, 'urlopen'): - urlresolver = urlgrabber.grabber - else: - import urllib - urlresolver = urllib - - scheme = urlparse.urlparse(mirrorlist)[0] - if scheme == '': - url = 'file://' + mirrorlist - else: - url = mirrorlist - - try: - fo = urlresolver.urlopen(url, proxies=pdict) - except urlgrabber.grabber.URLGrabError, e: - print "Could not retrieve mirrorlist %s error was\n%s" % (url, e) - fo = None - - if fo is not None: - content = fo.readlines() - for line in content: - if re.match('^\s*\#.*', line) or re.match('^\s*$', line): - continue - mirror = re.sub('\n$', '', line) # no more trailing \n's - (mirror, count) = re.subn('\$ARCH', '$BASEARCH', mirror) - returnlist.append(mirror) - - return returnlist - diff --git a/yum-presto/shared/prestoThread.py b/yum-presto/shared/prestoThread.py deleted file mode 100644 index e090910..0000000 --- a/yum-presto/shared/prestoThread.py +++ /dev/null @@ -1,53 +0,0 @@ -# authors: Ahmed Kamal <email.ahmedkamal@googlemail.com> -# Jonathan Dieter <jdieter@gmail.com> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# Copyright 2007 Ahmed Kamal, Jonathan Dieter - -import threading - -class ReconstructionThread(threading.Thread): - def __init__(self, queue, lock, run_function): - threading.Thread.__init__(self) - self.run_function = run_function - self.queue = queue - self.lock = lock - self.can_exit = False - self.messages = "" - - def run(self): - while True: - try: - retval = self.queue.get(not self.can_exit) - except: - # If we're done with our drpms and no more are coming, let's - # blow this joint - break - if retval != None: - messages = apply(self.run_function, retval) - if self.can_exit: - # If there are not going to be any more new drpms, - # send messages directly to conduit - conduit = retval[0] - if self.messages != "": - conduit.info(2, self.messages[:-1]) - self.messages = "" - if messages != "": - conduit.info(2, messages[:-1]) - else: - # We may be downloading drpms still, so queue messages - self.lock.acquire() - self.messages += messages - self.lock.release() diff --git a/yum-presto/shared/prestoTransaction.py b/yum-presto/shared/prestoTransaction.py deleted file mode 100644 index ffaad01..0000000 --- a/yum-presto/shared/prestoTransaction.py +++ /dev/null @@ -1,140 +0,0 @@ -# author: Jonathan Dieter <jdieter@gmail.com> -# -# format_number taken almost completely from progress_meter.py in yum -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# Copyright 2005 Duke University -# Copyright 2007 Jonathan Dieter - -import os -import deltarpm - -def format_number(number, SI=False, space=''): - """Turn numbers into human-readable metric-like numbers""" - symbols = ['', # (none) - 'K', # kilo - 'M', # mega - 'G', # giga - 'T', # tera - 'P', # peta - 'E', # exa - 'Z', # zetta - 'Y'] # yotta - - if SI: step = 1000.0 - else: step = 1024.0 - - thresh = 999 - depth = 0 - - # we want numbers between - while number > thresh: - depth = depth + 1 - number = number / step - - # just in case someone needs more than 1000 yottabytes! - diff = depth - len(symbols) + 1 - if diff > 0: - depth = depth - diff - number = number * thresh**depth - - if type(number) == type(1) or type(number) == type(1L): - format = '%i%s%s' - elif number < 9.95: - # must use 9.95 for proper sizing. For example, 9.99 will be - # rounded to 10.0 with the .1f format string (which is too long) - format = '%.1f%s%s' - else: - format = '%.0f%s%s' - - return(format % (number, space, symbols[depth])) - - -def find_available_drpms(conduit, newpkg): - """Find any applicable drpms for newpkg - newpkg is a TransactionMember""" - - rpmdb = conduit.getRpmDB() - - is_local = False - - # Set p_repo to be packages delta repository or set to False if - # there is no delta repository - try: - p_repo = newpkg.po.repo.p_repo - drpm_enabled = p_repo.enabled - - po = newpkg.po - if hasattr(po, 'pkgtype') and po.pkgtype == 'local': - is_local = True - else: - local = po.localPkg() - if os.path.exists(local): - cursize = os.stat(local)[6] - totsize = long(po.size) - if not po.verifyLocalPkg(): - if cursize >= totsize: # otherwise keep it around for regetting - os.unlink(local) - else: - conduit.info(5, "using local copy of %s" % po) - is_local = True - - except: - conduit.info(5, "No Presto repository information for %s.%s %i:%s-%s" % (newpkg.name, newpkg.arch, int(newpkg.epoch), newpkg.version, newpkg.release)) - drpm_enabled = False - is_local = False - - chosen_drpm = None - - # First part of key when matching drpms - key1 = "%s*%s*%i*%s*%s" % (newpkg.name, newpkg.arch, int(newpkg.epoch), newpkg.version, newpkg.release) - - # Find any installed packages that match the ones we want to download - installed = rpmdb.searchNevra(newpkg.name, None, None, None, newpkg.arch) - - if installed == []: - is_installed = False - else: - is_installed = True - - - if is_installed and drpm_enabled and not is_local: - for oldpkg in installed: - # Generate second part of key for matching drpms, then full key - key2 = "%s*%s*%i*%s*%s" % (oldpkg.name, oldpkg.arch, int(oldpkg.epoch), oldpkg.version, oldpkg.release) - key = "%s!!%s" % (key1, key2) - - # Check whether we have a matching drpm - if p_repo.deltalist.has_key(key): - # Check whether or not we already have a matching drpm, then choose smallest of the two if we do - if chosen_drpm == None or p_repo.deltalist[key]['size'] < chosen_drpm['size']: - - # Get sequence code for drpm - sequence = p_repo.deltalist[key]['sequence'] - drpm = deltarpm.DeltaRpmWrapper(conduit) - - # Attempt to apply sequence code for drpm. If this fails, drpm will not apply cleanly, so - # don't even try to download it. - try: - drpm.verifySequence(newpkg.po.arch, sequence) - except: - conduit.info(5, "Verification of %s failed" % sequence) - else: - chosen_drpm = p_repo.deltalist[key] - chosen_drpm['baseurl'] = p_repo.baseurl[0] - newpkg.po.oldpkg_string = "%s.%s %s:%s-%s" % (oldpkg.name, oldpkg.arch, oldpkg.epoch, oldpkg.version, oldpkg.release) - newpkg.po.newpkg_string = "%s.%s %s:%s-%s" % (newpkg.name, newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release) - - return (chosen_drpm, installed, is_local, drpm_enabled) diff --git a/yum-presto/shared/prestomdparser.py b/yum-presto/shared/prestomdparser.py deleted file mode 100644 index 6764f71..0000000 --- a/yum-presto/shared/prestomdparser.py +++ /dev/null @@ -1,169 +0,0 @@ -# author: Jonathan Dieter <jdieter@gmail.com> -# -# mostly taken from mdparser.py (part of yum) with a few minor modifications -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# Copyright 2005 Duke University -# Portions copyright 2007 Jonathan Dieter - -import gzip -try: - from cElementTree import iterparse -except: - from xml.etree.cElementTree import iterparse - -from cStringIO import StringIO - -#TODO: document everything here - -class PrestoMDParser: - - def __init__(self, filename): - - # Set up mapping of meta types to handler classes - handlers = { - '{http://linux.duke.edu/metadata/common}metadata': DeltasEntry, - } - - self.total = None - self.count = 0 - self._handlercls = None - - # Read in type, set package node handler and get total number of - # packages - if filename[-3:] == '.gz': fh = gzip.open(filename, 'r') - else: fh = open(filename, 'r') - parser = iterparse(fh, events=('start', 'end')) - self.reader = parser.__iter__() - event, elem = self.reader.next() - self._handlercls = handlers.get(elem.tag, None) - if not self._handlercls: - raise ValueError('Unknown repodata type "%s" in %s' % ( - elem.tag, filename)) - - def getDeltaList(self): - for event, elem in self.reader: - if event == 'end' and elem.tag == '{http://linux.duke.edu/metadata/common}metadata': - return self._handlercls(elem) - - -class BaseEntry: - def __init__(self, elem): - self._p = {} - - def __getitem__(self, k): - return self._p[k] - - def keys(self): - return self._p.keys() - - def values(self): - return self._p.values() - - def has_key(self, k): - return self._p.has_key(k) - - def __str__(self): - out = StringIO() - keys = self.keys() - keys.sort() - for k in keys: - line = u'%s=%s\n' % (k, self[k]) - out.write(line.encode('utf8')) - return out.getvalue() - - def _bn(self, qn): - if qn.find('}') == -1: return qn - return qn.split('}')[1] - - def _prefixprops(self, elem, prefix): - ret = {} - for key in elem.attrib.keys(): - ret[prefix + '_' + self._bn(key)] = elem.attrib[key] - return ret - -class DeltasEntry(BaseEntry): - def __init__(self, deltas): - BaseEntry.__init__(self, deltas) - # Avoid excess typing :) - p = self._p - - for elem in deltas: - temp = {} - key1 = "" - key2 = "" - for child in elem: - name = self._bn(child.tag) - if name in ('name', 'arch'): - temp[name] = child.text - - elif name == 'version': - attrib = child.attrib - try: - attrib['epoch'] = int(attrib['epoch']) - except: - attrib['epoch'] = 0 - key1 = "%s*%s*%i*%s*%s" % (temp['name'], temp['arch'], attrib['epoch'], attrib['ver'], attrib['rel']) - - elif name == 'deltas': - for oldrpm in child: - temp2 = {} - value = {} - key = None - for oldrpm_child in oldrpm: - name = self._bn(oldrpm_child.tag) - if name in ('name', 'arch'): - temp2[name] = oldrpm_child.text - - elif name == 'version': - ch_attrib = oldrpm_child.attrib - try: - ch_attrib['epoch'] = int(ch_attrib['epoch']) - except: - ch_attrib['epoch'] = attrib['epoch'] - try: - ch_attrib['ver'] = ch_attrib['ver'] - except: - ch_attrib['ver'] = attrib['ver'] - if not temp2.has_key('name'): - temp2['name'] = temp['name'] - if not temp2.has_key('arch'): - temp2['arch'] = temp['arch'] - key2 = "%s*%s*%i*%s*%s" % (temp2['name'], temp2['arch'], ch_attrib['epoch'], ch_attrib['ver'], ch_attrib['rel']) - key = "%s!!%s" % (key1, key2) - p[key] = {} - - if name in ('sequence', 'drpm_filename', 'size'): - p[key][name] = oldrpm_child.text - - if name == "checksum": - p[key][name] = oldrpm_child.text - p[key]["%s_type" % name] = oldrpm_child.attrib['type'] - deltas.clear() - -def test(): - import sys - - parser = PrestoMDParser(sys.argv[1]) - - deltalist = parser.getDeltaList() - - print '-' * 40 - print deltalist - - print 'read: %s deltarpms ' % (len(deltalist.keys())) - -if __name__ == '__main__': - test() |