From 4abf9bc75423551b2fb0a9e8eee53868c4fb875a Mon Sep 17 00:00:00 2001 From: Jonathan Dieter Date: Sat, 24 Mar 2007 19:48:54 +0200 Subject: A number of bug-fixes and added --disablepresto command-line option Signed-off-by: Jonathan Dieter --- ChangeLog | 12 +++ Changelog | 5 -- presto.conf | 3 + presto.py | 210 +++++++++++++++++++++++++++++++-------------------- shared/prestoRepo.py | 8 +- 5 files changed, 146 insertions(+), 92 deletions(-) create mode 100644 ChangeLog delete mode 100644 Changelog diff --git a/ChangeLog b/ChangeLog new file mode 100644 index 0000000..44c5044 --- /dev/null +++ b/ChangeLog @@ -0,0 +1,12 @@ +* Sat Mar 24 2007 Jonathan Dieter - 0.2.2 + - Fixed "not showing download error" bug + - Added --disablepresto yum command-line option + - Added code to trap the (hopefully) unlikely scenario where applydeltarpm + fails + - Show byte savings at end of yum update + +* Fri Mar 23 2007 Jonathan Dieter - 0.2.1 + - Fixed bug in handling mirrorlists in original repositories + +* Thu Mar 22 2007 Jonathan Dieter - 0.2.0 + - Initial release diff --git a/Changelog b/Changelog deleted file mode 100644 index 83c8d81..0000000 --- a/Changelog +++ /dev/null @@ -1,5 +0,0 @@ -* Fri Mar 23 2007 Jonathan Dieter - 0.2.1 -- Fixed bug in handling mirrorlists in original repositories - -* Thu Mar 22 2007 Jonathan Dieter - 0.2.0 -- Initial release diff --git a/presto.conf b/presto.conf index 89a18b6..899a4e2 100644 --- a/presto.conf +++ b/presto.conf @@ -4,3 +4,6 @@ neverkeepdeltas=1 [updates] deltaurl=http://www.lesbg.com/jdieter/updates/fc6/i386/ + +[extras] +deltaurl=http://www.lesbg.com/jdieter/extras/fc6/i386/ diff --git a/presto.py b/presto.py index 77d4dc2..f084aa6 100644 --- a/presto.py +++ b/presto.py @@ -33,98 +33,144 @@ import prestoTransaction requires_api_version = '2.1' plugin_type = (TYPE_INTERACTIVE,) -# Setup repository specific deltarpm url and mirrorlist -def config_hook(conduit): +rpm_size = 0 +drpm_size = 0 + +# Configuration stuff +def config_hook(conduit): + # Set up repository specific deltarpm url and mirrorlist config.RepoConf.deltaurl = config.UrlListOption() config.RepoConf.deltamirrorlist = config.UrlOption() + + # Add --disable-presto option + parser = conduit.getOptParser() + parser.add_option('', '--disablepresto', dest='disablepresto', + action='store_true', default=False, + help="disable Presto plugin and don't download any deltarpms") # Set up Presto repositories def prereposetup_hook(conduit): - conduit.info(2, 'Setting up Presto') - for active_repo in conduit.getRepos().listEnabled(): - p_repo = PrestoRepository(active_repo, conduit) - p_repo.setup(conduit.getConf().cache) + opts, commands = conduit.getCmdLine() + if not opts.disablepresto: + conduit.info(2, 'Setting up Presto') + for active_repo in conduit.getRepos().listEnabled(): + p_repo = PrestoRepository(active_repo, conduit) + p_repo.setup(conduit.getConf().cache) - conduit.info(2, 'Reading Presto metadata in from local files') - for active_repo in conduit.getRepos().listEnabled(): - xml = active_repo.p_repo.getPrestoXML() - if active_repo.p_repo.enabled: - xmldata = active_repo.p_repo.repoXML.getData('deltas') - (ctype, csum) = xmldata.checksum - parser = PrestoMDParser(xml) - active_repo.p_repo.deltalist = parser.getDeltaList() - - conduit.info(2, 'Setting up repositories') + conduit.info(2, 'Reading Presto metadata in from local files') + for active_repo in conduit.getRepos().listEnabled(): + xml = active_repo.p_repo.getPrestoXML() + if active_repo.p_repo.enabled: + xmldata = active_repo.p_repo.repoXML.getData('deltas') + (ctype, csum) = xmldata.checksum + parser = PrestoMDParser(xml) + active_repo.p_repo.deltalist = parser.getDeltaList() + + conduit.info(2, 'Setting up repositories') + else: + conduit.info(5, '--disablepresto specified - Presto disabled') def postresolve_hook(conduit): - # Cycle through packages to see if there's a deltarpm available - for newpkg in conduit.getTsInfo(): - if newpkg.ts_state != "e": - chosen_drpm = prestoTransaction.find_available_drpms(conduit, newpkg) + global rpm_size + global drpm_size + + opts, commands = conduit.getCmdLine() + if not opts.disablepresto: + # Cycle through packages to see if there's a deltarpm available + for newpkg in conduit.getTsInfo(): + if newpkg.ts_state != "e": + chosen_drpm = prestoTransaction.find_available_drpms(conduit, newpkg) - # If a drpm was found, change certain package information so it reflects - # the drpm, not the rpm. - if chosen_drpm != None: - newpkg.po.hasdrpm = True - newpkg.po.simple['basepath'] = chosen_drpm['baseurl'] - newpkg.po.simple['realpackagesize'] = newpkg.po.simple['packagesize'] - newpkg.po.simple['packagesize'] = chosen_drpm['size'] - newpkg.po.simple['realrelativepath'] = newpkg.po.simple['relativepath'] - newpkg.po.simple['relativepath'] = chosen_drpm['drpm_filename'] - newpkg.po.reallocalpath = newpkg.po.localpath - newpkg.po.localpath = newpkg.po.repo.deltasdir + "/" + os.path.basename(chosen_drpm['drpm_filename']) - newpkg.po.to = newpkg - newpkg.realpkgtup = newpkg.pkgtup - newpkg.pkgtup = (newpkg.name + " *", newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release) - for (csum_type, csum, csumid) in newpkg.po._checksums: - if csumid: - newpkg.po._realchecksum = (csum_type, csum, csumid) - newpkg.po._checksums.remove((csum_type, csum, csumid)) - csum_type = chosen_drpm['checksum_type'] - csum = chosen_drpm['checksum'] - newpkg.po._checksums.append((csum_type, csum, csumid)) + # If a drpm was found, change certain package information so it reflects + # the drpm, not the rpm. + if chosen_drpm != None: + rpm_size += int(newpkg.po.simple['packagesize']) + drpm_size += int(chosen_drpm['size']) + newpkg.po.hasdrpm = True + newpkg.po.simple['basepath'] = chosen_drpm['baseurl'] + newpkg.po.simple['realpackagesize'] = newpkg.po.simple['packagesize'] + newpkg.po.simple['packagesize'] = chosen_drpm['size'] + newpkg.po.simple['realrelativepath'] = newpkg.po.simple['relativepath'] + newpkg.po.simple['relativepath'] = chosen_drpm['drpm_filename'] + newpkg.po.reallocalpath = newpkg.po.localpath + newpkg.po.localpath = newpkg.po.repo.deltasdir + "/" + os.path.basename(chosen_drpm['drpm_filename']) + newpkg.po.to = newpkg + newpkg.realpkgtup = newpkg.pkgtup + newpkg.pkgtup = (newpkg.name + " *", newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release) + for (csum_type, csum, csumid) in newpkg.po._checksums: + if csumid: + newpkg.po._realchecksum = (csum_type, csum, csumid) + newpkg.po._checksums.remove((csum_type, csum, csumid)) + csum_type = chosen_drpm['checksum_type'] + csum = chosen_drpm['checksum'] + newpkg.po._checksums.append((csum_type, csum, csumid)) - conduit.info(2, "Found deltarpm update for %s.%s %s:%s.%s" % (newpkg.name, newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release)) - else: - newpkg.po.hasdrpm = False - return - - # Free up memory used by deleting Presto repositories - for active_repo in conduit.getRepos().listEnabled(): - if active_repo.p_repo.enabled: - del active_repo.p_repo + conduit.info(2, "Found deltarpm update for %s.%s %s:%s.%s" % (newpkg.name, newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release)) + else: + newpkg.po.hasdrpm = False + if newpkg.po.repo.p_repo.enabled: + rpm_size += int(newpkg.po.simple['packagesize']) + drpm_size += int(newpkg.po.simple['packagesize']) + + return + + # Free up memory used by deleting Presto repositories + for active_repo in conduit.getRepos().listEnabled(): + if active_repo.p_repo.enabled: + del active_repo.p_repo def postdownload_hook(conduit): - # Cycle through packages to see if we've downloaded a deltarpm - conduit.info(2, "Rebuilding full packages from deltas") - for pkg in conduit.getDownloadPackages(): - if pkg.hasdrpm and pkg.verifyLocalPkg(): - # Apply deltarpm and save where rpm would have been saved - drpm = deltarpm.DeltaRpmWrapper(conduit) - drpm.apply(pkg.reallocalpath, pkg.localpath) - drpm_path = pkg.localpath - - # Change package information to reflect original rpm information - pkg.to.pkgtup = pkg.to.realpkgtup - pkg.localpath = pkg.reallocalpath - pkg.simple['packagesize'] = pkg.simple['realpackagesize'] - del pkg.simple['basepath'] - for (csum_type, csum, csumid) in pkg._checksums: - if csumid: - pkg._checksums.remove((csum_type, csum, csumid)) - pkg._checksums.append(pkg._realchecksum) - - # Check to see whether or not we should keep the drpms - # FIXME: Is there any way to see whether or not a Boolean option was not set? - if conduit.confBool('main', 'neverkeepdeltas'): - delete = True - elif conduit.confBool('main', 'keepdeltas'): - delete = False - elif conduit.getConf().keepcache != 0: - delete = False - else: - delete = True - - if delete: - os.unlink(drpm_path) + opts, commands = conduit.getCmdLine() + if not opts.disablepresto: + failure = False + # Cycle through packages to see if we've downloaded a deltarpm + conduit.info(2, "Rebuilding full packages from deltas") + for pkg in conduit.getDownloadPackages(): + if pkg.hasdrpm and pkg.verifyLocalPkg(): + # Apply deltarpm and save where rpm would have been saved + drpm = deltarpm.DeltaRpmWrapper(conduit) + this_failure = False + try: + drpm.apply(pkg.reallocalpath, pkg.localpath) + except: + failure = True + this_failure = True + if not this_failure: + drpm_path = pkg.localpath + + # Change package information to reflect original rpm information + pkg.to.pkgtup = pkg.to.realpkgtup + pkg.localpath = pkg.reallocalpath + pkg.simple['packagesize'] = pkg.simple['realpackagesize'] + del pkg.simple['basepath'] + for (csum_type, csum, csumid) in pkg._checksums: + if csumid: + pkg._checksums.remove((csum_type, csum, csumid)) + pkg._checksums.append(pkg._realchecksum) + + # Check to see whether or not we should keep the drpms + # FIXME: Is there any way to see whether or not a Boolean option was not set? + if conduit.confBool('main', 'neverkeepdeltas'): + delete = True + elif conduit.confBool('main', 'keepdeltas'): + delete = False + elif conduit.getConf().keepcache != 0: + delete = False + else: + delete = True + + if delete: + os.unlink(drpm_path) + + if failure: + raise PluginYumExit("Error rebuilding at least one deltarpm. Please run report this error to\nhttps://hosted.fedoraproject.org/projects/presto/newticket. To bypass this problem, run yum \nwith the --disablepresto option") + +def posttrans_hook(conduit): + global rpm_size + global drpm_size + + if rpm_size > 0: + conduit.info(2, "Size of all files downloaded from Presto-enabled repositories: %i bytes" % drpm_size) + conduit.info(2, "Size that would have been downloaded if Presto wasn't enabled: %i bytes" % rpm_size) + conduit.info(2, "This is a savings of %i percent" % (100 - ((drpm_size * 100) / rpm_size))) diff --git a/shared/prestoRepo.py b/shared/prestoRepo.py index 6ad4b99..f9c4648 100644 --- a/shared/prestoRepo.py +++ b/shared/prestoRepo.py @@ -113,9 +113,9 @@ class PrestoRepository(Repository): self.metadata_expire = repo.metadata_expire self.basecachedir = repo.basecachedir self.callback = repo.callback - self.failure_obj = None - self.mirror_failure_obj = None - self.interrupt_callback = None + self.failure_obj = repo.failure_obj + self.mirror_failure_obj = repo.mirror_failure_obj + self.interrupt_callback = repo.interrupt_callback self.drpm_list = {} self.parent = repo repo.p_repo = self @@ -238,7 +238,6 @@ class PrestoRepository(Repository): http_headers=headers, reget='simple') - self.grab = mgclass(self.grabfunc, self.urls, failure_callback=self.mirror_failure_obj) @@ -326,7 +325,6 @@ class PrestoRepository(Repository): raise Errors.RepoError, \ "Caching enabled but no local cache of %s from %s" % (local, self) - if url is not None: ug = URLGrabber(keepalive = self.keepalive, bandwidth = self.bandwidth, -- cgit