diff options
| -rw-r--r-- | ChangeLog | 6 | ||||
| -rw-r--r-- | Makefile | 1 | ||||
| -rwxr-xr-x | makerepo/createprestorepo.py | 8 | ||||
| -rw-r--r-- | presto.py | 104 | ||||
| -rw-r--r-- | shared/deltarpm.py | 2 | ||||
| -rw-r--r-- | shared/prestoDownload.py | 169 | ||||
| -rw-r--r-- | shared/prestoRepo.py | 16 | ||||
| -rw-r--r-- | shared/prestoTransaction.py | 18 |
8 files changed, 244 insertions, 80 deletions
@@ -1,3 +1,9 @@ +* Wed Mar 28 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.0 + - Massive changes to downloading structure + - When unable to rebuild drpm, we now download full rpm + - Stop doing slow MD5 check and just check RPM header + while we have a prelink bug + * Mon Mar 26 2007 Jonathan Dieter <jdieter@gmail.com> - 0.2.9 - Fix another mirrorlist bug - Minor optimization @@ -12,4 +12,5 @@ install: install -m 644 shared/prestomdparser.py $(DESTDIR)/usr/share/presto install -m 644 shared/prestoTransaction.py $(DESTDIR)/usr/share/presto install -m 644 shared/prestoLog.py $(DESTDIR)/usr/share/presto + install -m 644 shared/prestoDownload.py $(DESTDIR)/usr/share/presto install -m 644 shared/deltarpm.py $(DESTDIR)/usr/share/presto diff --git a/makerepo/createprestorepo.py b/makerepo/createprestorepo.py index 0fee5b4..a217045 100755 --- a/makerepo/createprestorepo.py +++ b/makerepo/createprestorepo.py @@ -23,7 +23,7 @@ import dumpMetadata from dumpMetadata import _gzipOpen, getChecksum #### import Utils -DEBUG = False +DEBUG = True #### Utils.setdebug(DEBUG) SUFFIX='drpm' @@ -138,7 +138,7 @@ def genDeltaRPM(ts, newrpm, oldrpm, is_new_package, srcdir, dstdir, locroot): XML_oldrpm(locroot, deltaRPMName, oldrpm, newrpm, sequence, dsize) if DEBUG: print "DEBUG skipping %s" % (deltaRPMName) - elif os.path.exists("%s%s.dontdelta" % (srcdir, deltaRPMName)) or os.path.getsize(f1) > 70000000: + elif os.path.exists("%s%s.dontdelta" % (srcdir, deltaRPMName)): pass else: deltaCommand = 'makedeltarpm -s %s%s.seq %s %s %s%s' % (srcdir, deltaRPMName, f2, f1, srcdir, deltaRPMName) @@ -208,7 +208,7 @@ def createPrestoRepo(srcdir, dstdir): print ' Nothing found.' return changed assert srcfiles[0].startswith(srcdir) - + # Check whether dstdir exists, and if it doesn't, create it if not os.access(dstdir, os.F_OK): os.makedirs(dstdir, 0755) @@ -264,7 +264,7 @@ def createPrestoRepo(srcdir, dstdir): # Generate delta rpm is_new_package = True locroot = xmlroot - for rpm in l[1:]: + for rpm in l[1:]: (is_new_package, locroot) = genDeltaRPM(ts, l[0], rpm, is_new_package, srcdir, dstdir, locroot) if not len(srcfiles): @@ -30,6 +30,7 @@ from prestoRepo import PrestoRepository from prestomdparser import PrestoMDParser import prestoTransaction import prestoLog +import prestoDownload requires_api_version = '2.1' LOG_FILE = "/var/log/presto.log" @@ -37,7 +38,7 @@ plugin_type = (TYPE_INTERACTIVE,) rpm_size = 0 drpm_size = 0 -was_drpm = False +drpm_count = 0 # Configuration stuff def config_hook(conduit): @@ -75,7 +76,7 @@ def postreposetup_hook(conduit): def postresolve_hook(conduit): global rpm_size global drpm_size - global was_drpm + global drpm_count opts, commands = conduit.getCmdLine() if not opts.disablepresto: @@ -87,31 +88,24 @@ def postresolve_hook(conduit): # If a drpm was found, change certain package information so it reflects # the drpm, not the rpm. if chosen_drpm != None: - was_drpm = True - rpm_size += int(newpkg.po.simple['packagesize']) + newpkg.po.has_drpm = True + conduit.info(2, "Found deltarpm update for %s.%s %s:%s-%s" % (newpkg.name, newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release)) + # In yum 3.0.x, this doesn't get defined if you run "yum update x" rather than "yum update" + rpm_size += int(newpkg.po.size) drpm_size += int(chosen_drpm['size']) - newpkg.po.hasdrpm = True - newpkg.po.simple['basepath'] = chosen_drpm['baseurl'] - newpkg.po.simple['realpackagesize'] = newpkg.po.simple['packagesize'] + newpkg.po.simple['realpackagesize'] = newpkg.po.size newpkg.po.simple['packagesize'] = chosen_drpm['size'] - newpkg.po.simple['realrelativepath'] = newpkg.po.simple['relativepath'] - newpkg.po.simple['relativepath'] = chosen_drpm['drpm_filename'] - newpkg.po.reallocalpath = newpkg.po.localpath - newpkg.po.localpath = newpkg.po.repo.deltasdir + "/" + os.path.basename(chosen_drpm['drpm_filename']) + newpkg.po.simple['deltasize'] = chosen_drpm['size'] + newpkg.po.simple['deltarelativepath'] = chosen_drpm['drpm_filename'] + newpkg.po.simple['deltachecksumtype'] = chosen_drpm['checksum_type'] + newpkg.po.simple['deltachecksum'] = chosen_drpm['checksum'] + newpkg.po.simple['deltalocalpath'] = newpkg.po.repo.deltasdir + "/" + os.path.basename(chosen_drpm['drpm_filename']) newpkg.po.to = newpkg newpkg.realpkgtup = newpkg.pkgtup newpkg.pkgtup = (newpkg.name + " *", newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release) - for (csum_type, csum, csumid) in newpkg.po._checksums: - if csumid: - newpkg.po._realchecksum = (csum_type, csum, csumid) - newpkg.po._checksums.remove((csum_type, csum, csumid)) - csum_type = chosen_drpm['checksum_type'] - csum = chosen_drpm['checksum'] - newpkg.po._checksums.append((csum_type, csum, csumid)) - - conduit.info(2, "Found deltarpm update for %s.%s %s:%s.%s" % (newpkg.name, newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release)) + newpkg.po.hasdrpm = True + drpm_count += 1 else: - newpkg.po.hasdrpm = False if installed and drpm_enabled and not local: try: rpm_size += int(newpkg.po.simple['packagesize']) @@ -120,59 +114,27 @@ def postresolve_hook(conduit): pass return - # Free up memory used by deleting Presto repositories - for active_repo in conduit.getRepos().listEnabled(): - if active_repo.p_repo.enabled: - del active_repo.p_repo -def postdownload_hook(conduit): - global was_drpm +def predownload_hook(conduit): + global drpm_count + + pkglist = conduit.getDownloadPackages() opts, commands = conduit.getCmdLine() - if not opts.disablepresto and was_drpm: - failure = False - # Cycle through packages to see if we've downloaded a deltarpm - conduit.info(2, "Rebuilding full packages from deltas") - for pkg in conduit.getDownloadPackages(): - if pkg.hasdrpm and pkg.verifyLocalPkg(): - # Apply deltarpm and save where rpm would have been saved - drpm = deltarpm.DeltaRpmWrapper(conduit) - this_failure = False - try: - drpm.apply(pkg.reallocalpath, pkg.localpath) - except: - conduit.info(2, "Error rebuilding rpm from %s!" % pkg.localpath) - failure = True - this_failure = True - if not this_failure: - drpm_path = pkg.localpath - - # Change package information to reflect original rpm information - pkg.to.pkgtup = pkg.to.realpkgtup - pkg.localpath = pkg.reallocalpath - pkg.simple['packagesize'] = pkg.simple['realpackagesize'] - del pkg.simple['basepath'] - for (csum_type, csum, csumid) in pkg._checksums: - if csumid: - pkg._checksums.remove((csum_type, csum, csumid)) - pkg._checksums.append(pkg._realchecksum) - - # Check to see whether or not we should keep the drpms - # FIXME: Is there any way to see whether or not a Boolean option was not set? - if conduit.confBool('main', 'neverkeepdeltas'): - delete = True - elif conduit.confBool('main', 'keepdeltas'): - delete = False - elif conduit.getConf().keepcache != 0: - delete = False - else: - delete = True - - if delete: - os.unlink(drpm_path) - - if failure: - raise PluginYumExit("Error rebuilding at least one deltarpm. Please report this error to\nhttp://hosted.fedoraproject.org/projects/presto/newticket. To bypass this problem, run yum \nwith the --disablepresto option") + if not opts.disablepresto and drpm_count > 0: + # Download deltarpms + problems = prestoDownload.downloadPkgs(conduit, pkglist) + + # If 'exitondownloaderror' is on, exit + if conduit.confBool('main', 'exitondownloaderror') and len(problems.keys()) > 0: + errstring = '' + errstring += 'Error Downloading Packages:\n' + for key in problems.keys(): + errors = misc.unique(problems[key]) + for error in errors: + errstring += ' %s: %s\n' % (key, error) + raise PluginYumExit(errstring) + def posttrans_hook(conduit): global rpm_size diff --git a/shared/deltarpm.py b/shared/deltarpm.py index a2aea2e..710a8bb 100644 --- a/shared/deltarpm.py +++ b/shared/deltarpm.py @@ -80,7 +80,7 @@ class DeltaRpmWrapper: constructs file names and paths based on given RpmDescription and instance settings for directories""" self.conduit.info(7, '%s.verify(%s)' % (self.__class__, sequence)) p = Process(self.conduit) - p.run(APPLY, '-c', '-s', sequence) + p.run(APPLY, '-s', sequence) if p.returnCode(): # in case of error, raise exception raise Exception("Could not verify sequence of deltarpm: %d" % (p.returnCode())) diff --git a/shared/prestoDownload.py b/shared/prestoDownload.py new file mode 100644 index 0000000..45318ba --- /dev/null +++ b/shared/prestoDownload.py @@ -0,0 +1,169 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# Copyright 2005 Duke University +# Copyright 2007 Jonathan Dieter + +import os +from yum import misc +from yum import Errors +from yum import types +from urlgrabber.grabber import URLGrabError +import deltarpm + +def verifyDelta(fo, po, conduit, raiseError): + """verifies the deltarpm is what we expect it to be + raiseError = defaults to 0 - if 1 then will raise + a URLGrabError if the file does not check out. + otherwise it returns false for a failure, true for success""" + + if type(fo) is types.InstanceType: + fo = fo.filename + + try: + verifyChecksum(fo, po.returnSimple('deltachecksumtype'), po.returnSimple('deltachecksum')) + except: + if raiseError: + raise URLGrabError(-1, 'Package does not match intended download') + else: + return False + + return True + + +def verifyChecksum(filename, checksumType, csum): + """Verify the checksum of the file versus the + provided checksum""" + + try: + + filesum = misc.checksum(checksumType, filename) + except Errors.MiscError, e: + raise URLGrabError(-3, 'Could not perform checksum') + + if filesum != csum: + raise URLGrabError(-1, 'Package does not match checksum') + + return 0 + + +def downloadPkgs(conduit, pkglist): + """download list of package objects handed to you, return errors""" + + opts, commands = conduit.getCmdLine() + + errors = {} + def adderror(po, msg): + errors.setdefault(po, []).append(msg) + + # Check whether drpm is already downloaded + repo_cached = False + remote_pkgs = [] + rebuild_pkgs = [] + for po in conduit.getDownloadPackages(): + if hasattr(po, 'has_drpm') and po.has_drpm: + po.to.pkgtup = po.to.realpkgtup + local = po.returnSimple('deltalocalpath') + if os.path.exists(local): + cursize = os.stat(local)[6] + totsize = long(po.returnSimple('deltasize')) + try: + verifyChecksum(local, po.returnSimple('deltachecksumtype'), po.returnSimple('deltachecksum')) + except: + if po.repo.p_repo.cache: + repo_cached = True + adderror(po, 'package fails checksum but caching is ' + 'enabled for %s' % po.repo.p_repo.id) + + if cursize >= totsize: # otherwise keep it around for regetting + os.unlink(local) + else: + # Deltarpm is local and good, let's put it in the rebuild list + conduit.info(5, "using local copy of deltarpm for %s" % po) + rebuild_pkgs.append(po) + continue + remote_pkgs.append(po) + + # Download deltarpms + i = 0 + for po in remote_pkgs: + i += 1 + checkfunc = (verifyDelta, (po, conduit, 1), {}) + cache = po.repo.p_repo.http_caching != 'none' + dirstat = os.statvfs(po.repo.deltasdir) + if (dirstat.f_bavail * dirstat.f_bsize) <= long(po.size): + adderror(po, 'Insufficient space in download directory %s ' + 'to download' % po.repo.deltasdir) + continue + po.simple['reallocalpath'] = po.localpath + po.localpath = po.returnSimple('deltalocalpath') + po.simple['realrelativepath'] = po.returnSimple('relativepath') + po.simple['relativepath'] = po.returnSimple('deltarelativepath') + try: + text = '(%s/%s): %s' % (i, len(remote_pkgs), os.path.basename(po.returnSimple('deltarelativepath'))) + deltalocal = po.repo.p_repo.getPackage(po, checkfunc=checkfunc, text=text, cache=cache) + except Errors.RepoError, e: + adderror(po, str(e)) + else: + rebuild_pkgs.append(po) + + if errors.has_key(po): + del errors[po] + + po.simple['relativepath'] = po.returnSimple('realrelativepath') + po.localpath = po.returnSimple('reallocalpath') + if po.simple.has_key('realpackagesize'): + po.simple['packagesize'] = po.returnSimple('realpackagesize') + del po.simple['realpackagesize'] + del po.simple['realrelativepath'] + del po.simple['reallocalpath'] + po.simple['deltalocalpath'] = deltalocal + + # Rebuild rpms from downloaded deltarpms + for po in rebuild_pkgs: + deltalocal = po.returnSimple('deltalocalpath') + drpm = deltarpm.DeltaRpmWrapper(conduit) + try: + conduit.info(2, "Building %s from %s" % (os.path.basename(po.localpath), os.path.basename(deltalocal))) + drpm.apply(po.localpath, deltalocal) + except: + conduit.info(2, "Error rebuilding rpm from %s! Will download full package." % os.path.basename(deltalocal)) + try: + os.unlink(po.localpath) + except: + pass + else: + # Set package type to local, so yum doesn't try to download it later + po.pkgtype = "local" + + # Check to see whether or not we should keep the drpms + # FIXME: Is there any way to see whether or not a Boolean option was not set? + if conduit.confBool('main', 'neverkeepdeltas'): + delete = True + elif conduit.confBool('main', 'keepdeltas'): + delete = False + elif conduit.getConf().keepcache != 0: + delete = False + else: + delete = True + + if delete: + try: + os.unlink(deltalocal) + except: + pass + + return errors + + diff --git a/shared/prestoRepo.py b/shared/prestoRepo.py index f9c4648..582dc2f 100644 --- a/shared/prestoRepo.py +++ b/shared/prestoRepo.py @@ -158,11 +158,11 @@ class PrestoRepository(Repository): def dump(self): output = '[%s]\n' % self.id - vars = ['name', 'bandwidth', 'enabled', + vars = ['id', 'bandwidth', 'enabled', 'keepalive', 'proxy', 'proxy_password', 'proxy_username', 'retries', 'throttle', 'timeout', 'mirrorlist', - 'cachedir', 'deltasdir' ] + 'cachedir' ] vars.sort() for attr in vars: output = output + '%s = %s\n' % (attr, getattr(self, attr)) @@ -367,6 +367,18 @@ class PrestoRepository(Repository): return result + def getPackage(self, package, checkfunc = None, text = None, cache = True): + remote = package.returnSimple('relativepath') + local = package.localPkg() + basepath = package.returnSimple('basepath') + + return self.__get(url=basepath, + relative=remote, + local=local, + checkfunc=checkfunc, + text=text, + cache=cache + ) def metadataCurrent(self): """Check if there is a metadata_cookie and check its age. If the diff --git a/shared/prestoTransaction.py b/shared/prestoTransaction.py index 71b90f3..3d387a4 100644 --- a/shared/prestoTransaction.py +++ b/shared/prestoTransaction.py @@ -24,15 +24,29 @@ def find_available_drpms(conduit, newpkg): rpmdb = conduit.getRpmDB() + is_local = False + # Set p_repo to be packages delta repository or set to False if # there is no delta repository try: p_repo = newpkg.po.repo.p_repo drpm_enabled = p_repo.enabled - if os.path.exists(newpkg.po.localpath): + + po = newpkg.po + if hasattr(po, 'pkgtype') and po.pkgtype == 'local': is_local = True else: - is_local = False + local = po.localPkg() + if os.path.exists(local): + cursize = os.stat(local)[6] + totsize = long(po.size) + if not po.verifyLocalPkg(): + if cursize >= totsize: # otherwise keep it around for regetting + os.unlink(local) + else: + conduit.info(5, "using local copy of %s" % po) + is_local = True + except: conduit.info(5, "No Presto repository information for %s.%s %i:%s-%s" % (newpkg.name, newpkg.arch, int(newpkg.epoch), newpkg.version, newpkg.release)) drpm_enabled = False |