diff options
| -rw-r--r-- | yum-presto/ChangeLog | 4 | ||||
| -rw-r--r-- | yum-presto/presto.py | 32 | ||||
| -rw-r--r-- | yum-presto/shared/prestoDownload.py | 15 | ||||
| -rw-r--r-- | yum-presto/shared/prestoLog.py | 113 | ||||
| -rw-r--r-- | yum-presto/shared/prestoThread.py | 3 | ||||
| -rw-r--r-- | yum-presto/shared/prestoTransaction.py | 51 |
6 files changed, 155 insertions, 63 deletions
diff --git a/yum-presto/ChangeLog b/yum-presto/ChangeLog index 4612d88..4b6b95f 100644 --- a/yum-presto/ChangeLog +++ b/yum-presto/ChangeLog @@ -1,3 +1,7 @@ +* Wed Apr 4 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.5 + - Many small bug fixes + - Improved logging - Large changes + * Tue Apr 3 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.4 - Add patch from Ahmed Kamal to put rebuilding of rpms into a different thread diff --git a/yum-presto/presto.py b/yum-presto/presto.py index e9908ee..a5bd77e 100644 --- a/yum-presto/presto.py +++ b/yum-presto/presto.py @@ -17,6 +17,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2005 Duke University +# Copyright 2007 Jonathan Dieter from yum.plugins import TYPE_INTERACTIVE, PluginYumExit from yum import config @@ -39,9 +40,12 @@ plugin_type = (TYPE_INTERACTIVE,) rpm_size = 0 drpm_size = 0 drpm_count = 0 +log = None # Configuration stuff -def config_hook(conduit): +def config_hook(conduit): + global log + # Set up repository specific deltarpm url and mirrorlist config.RepoConf.deltaurl = config.UrlListOption() config.RepoConf.deltamirrorlist = config.UrlOption() @@ -51,6 +55,9 @@ def config_hook(conduit): parser.add_option('', '--disablepresto', dest='disablepresto', action='store_true', default=False, help="disable Presto plugin and don't download any deltarpms") + + # Set up logging + log = prestoLog.PrestoLog(conduit, LOG_FILE) # Set up Presto repositories def postreposetup_hook(conduit): @@ -117,33 +124,40 @@ def postresolve_hook(conduit): def predownload_hook(conduit): global drpm_count + global log pkglist = conduit.getDownloadPackages() opts, commands = conduit.getCmdLine() if not opts.disablepresto and drpm_count > 0: + conduit.info(2, "Downloading DeltaRPMs:") + # Download deltarpms - problems = prestoDownload.downloadPkgs(conduit, pkglist) + problems = prestoDownload.downloadPkgs(conduit, pkglist, log) # If 'exitondownloaderror' is on, exit if conduit.confBool('main', 'exitondownloaderror') and len(problems.keys()) > 0: - errstring = '' - errstring += 'Error Downloading Packages:\n' + errstring = 'Error Downloading Packages:\n' for key in problems.keys(): errors = misc.unique(problems[key]) for error in errors: errstring += ' %s: %s\n' % (key, error) raise PluginYumExit(errstring) + else: + conduit.info(2, "Downloading RPMs:") def posttrans_hook(conduit): global rpm_size global drpm_size - global LOG_FILE + global log + + log.close() if rpm_size > 0: - prestoLog.log(conduit, LOG_FILE, rpm_size, drpm_size) - - conduit.info(2, "Size of all updates downloaded from Presto-enabled repositories: %i bytes" % drpm_size) - conduit.info(2, "Size of updates that would have been downloaded if Presto wasn't enabled: %i bytes" % rpm_size) + drpm_string = prestoTransaction.format_number(drpm_size) + rpm_string = prestoTransaction.format_number(rpm_size) + + conduit.info(2, "Size of all updates downloaded from Presto-enabled repositories: %s" % drpm_string) + conduit.info(2, "Size of updates that would have been downloaded if Presto wasn't enabled: %s" % rpm_string) conduit.info(2, "This is a savings of %i percent" % (100 - ((drpm_size * 100) / rpm_size))) diff --git a/yum-presto/shared/prestoDownload.py b/yum-presto/shared/prestoDownload.py index f21d409..30e7514 100644 --- a/yum-presto/shared/prestoDownload.py +++ b/yum-presto/shared/prestoDownload.py @@ -60,7 +60,7 @@ def verifyChecksum(filename, checksumType, csum): return 0 -def reconstruct(conduit, po): +def reconstruct(conduit, po, log): deltalocal = po.returnSimple('deltalocalpath') retlist = "" @@ -75,7 +75,10 @@ def reconstruct(conduit, po): except: pass else: - retlist += "Built %s from deltarpm\n" % (os.path.basename(po.localpath), os.path.basename(deltalocal)) + # log success + log.log(po.oldpkg_string, po.newpkg_string, os.stat(po.localpath)[6], os.stat(deltalocal)[6]) + + #retlist += "Built %s from deltarpm\n" % os.path.basename(po.localpath) # Check to see whether or not we should keep the drpms # FIXME: Is there any way to see whether or not a Boolean option was not set? if conduit.confBool('main', 'neverkeepdeltas'): @@ -96,7 +99,7 @@ def reconstruct(conduit, po): -def downloadPkgs(conduit, pkglist): +def downloadPkgs(conduit, pkglist, log): """download list of package objects handed to you, return errors""" opts, commands = conduit.getCmdLine() @@ -134,7 +137,7 @@ def downloadPkgs(conduit, pkglist): else: # Deltarpm is local and good, let's put it in the rebuild thread. conduit.info(5, "using local copy of deltarpm for %s" % po) - queue.put((conduit, po)) + queue.put((conduit, po, log)) continue remote_pkgs.append(po) @@ -159,7 +162,7 @@ def downloadPkgs(conduit, pkglist): except Errors.RepoError, e: adderror(po, str(e)) else: - queue.put((conduit, po)) + queue.put((conduit, po, log)) po.simple['deltalocalpath'] = deltalocal if errors.has_key(po): @@ -180,6 +183,8 @@ def downloadPkgs(conduit, pkglist): curthread.messages = "" lock.release() + conduit.info(2, "Rebuilding rpms from deltarpms") + # Tell build thread that there are no more drpms and wait for it to exit curthread.can_exit = True curthread.join() diff --git a/yum-presto/shared/prestoLog.py b/yum-presto/shared/prestoLog.py index 3c0c1e6..1323346 100644 --- a/yum-presto/shared/prestoLog.py +++ b/yum-presto/shared/prestoLog.py @@ -15,57 +15,78 @@ # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2005 Duke University -def log(conduit, LOG_FILE, rpm_size, drpm_size): - # Open log file for reading - try: - log_file = open(LOG_FILE, "r") - log_exists = True - except: - conduit.info(5, "Info: %s doesn't exist. Will create." % LOG_FILE) - log_exists = False - - # Log file doesn't exist, create - if not log_exists: +class PrestoLog: + def __init__(self, conduit, log_filename): + # Open log file for reading try: - log_file = open(LOG_FILE, "w") - log_file.write("Download Size (without DRPM),Download Size (with DRPM),Percentage Savings,Total Percentage Savings\n") - log_file.close() + log_file = open(log_filename, "r") log_exists = True except: - conduit.info(2, "Warning: Unable to write to %s" % LOG_FILE) + conduit.info(7, "Info: %s doesn't exist. Will create." % log_filename) + log_exists = False + + # Log file doesn't exist, create + if not log_exists: + try: + log_file = open(log_filename, "w") + log_file.close() + log_exists = True + except: + conduit.info(2, "Warning: Unable to write to %s" % log_filename) + if log_exists: + try: + log_file = open(log_filename, "r") + except: + conduit.info(2, "Warning: Unable to open %s for reading." % log_filename) + log_exists = False + + # Cycle through items already in log so we can come up with total savings if log_exists: + self.total_rpm_size = 0 + self.total_drpm_size = 0 + + # Get rid of header line + log_file.readline() + + data = log_file.readline() + while data != "": + fc = data.rfind("-") + sc = data.rfind("-", 0, fc-1) + tc = data.rfind("-", 0, sc-1) + lc = data.rfind("-", 0, tc-1) + if lc != -1 and tc != -1 and sc != -1 and fc != -1: + self.total_rpm_size += int(data[lc+1:tc]) + self.total_drpm_size += int(data[tc+1:sc]) + data = log_file.readline() + log_file.close() + try: - log_file = open(LOG_FILE, "r") + log_file = open(log_filename, "a") except: - conduit.info(2, "Warning: Unable to open %s for reading." % LOG_FILE) - log_exists = False + conduit.info(2, "Warning: Unable to open %s for writing." % log_filename) + self.log_file = None + else: + self.log_file = log_filename + log_file.close() + + def log(self, oldrpm_name, newrpm_name, rpm_size, drpm_size): + # Write data to log + self.total_rpm_size += rpm_size + self.total_drpm_size += drpm_size + if self.log_file != None: + try: + log_file = open(self.log_file, "a") + except: + pass + else: + log_file.write("%s => %s - %i - %i - %i - %i\n" % (oldrpm_name, newrpm_name, rpm_size, drpm_size, 100 - ((drpm_size * 100) / rpm_size), 100 - ((self.total_drpm_size * 100) / self.total_rpm_size))) + log_file.close() + - # Cycle through items already in log so we can come up with total savings - if log_exists: - total_rpm_size = 0 - total_drpm_size = 0 - - # Get rid of header line - log_file.readline() - - data = log_file.readline() - while data != "": - fc = data.find(",") - sc = data.find(",", fc + 1) - total_rpm_size += int(data[:fc]) - total_drpm_size += int(data[fc + 1:sc]) - data = log_file.readline() - log_file.close() - total_rpm_size += rpm_size - total_drpm_size += drpm_size + def close(self): + if self.log_file != None: + try: + self.log_file.close() + except: + pass - try: - log_file = open(LOG_FILE, "a") - except: - conduit.info(2, "Warning: Unable to open %s for writing." % LOG_FILE) - log_exists = False - - # Write data to log - if log_exists: - log_file.write("%i,%i,%i,%i\n" % (rpm_size, drpm_size, 100 - ((drpm_size * 100) / rpm_size), 100 - ((total_drpm_size * 100) / total_rpm_size))) - log_file.close() diff --git a/yum-presto/shared/prestoThread.py b/yum-presto/shared/prestoThread.py index e67db76..e090910 100644 --- a/yum-presto/shared/prestoThread.py +++ b/yum-presto/shared/prestoThread.py @@ -44,7 +44,8 @@ class ReconstructionThread(threading.Thread): if self.messages != "": conduit.info(2, self.messages[:-1]) self.messages = "" - conduit.info(2, messages[:-1]) + if messages != "": + conduit.info(2, messages[:-1]) else: # We may be downloading drpms still, so queue messages self.lock.acquire() diff --git a/yum-presto/shared/prestoTransaction.py b/yum-presto/shared/prestoTransaction.py index 166fe6d..4f08391 100644 --- a/yum-presto/shared/prestoTransaction.py +++ b/yum-presto/shared/prestoTransaction.py @@ -1,5 +1,7 @@ # author: Jonathan Dieter <jdieter@gmail.com> # +# format_number taken almost completely from progress_meter.py in yum +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or @@ -14,10 +16,52 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Copyright 2005 Duke University +# Copyright 2007 Jonathan Dieter import os import deltarpm +def format_number(number, SI=False, space=''): + """Turn numbers into human-readable metric-like numbers""" + symbols = ['', # (none) + 'K', # kilo + 'M', # mega + 'G', # giga + 'T', # tera + 'P', # peta + 'E', # exa + 'Z', # zetta + 'Y'] # yotta + + if SI: step = 1000.0 + else: step = 1024.0 + + thresh = 999 + depth = 0 + + # we want numbers between + while number > thresh: + depth = depth + 1 + number = number / step + + # just in case someone needs more than 1000 yottabytes! + diff = depth - len(symbols) + 1 + if diff > 0: + depth = depth - diff + number = number * thresh**depth + + if type(number) == type(1) or type(number) == type(1L): + format = '%i%s%s' + elif number < 9.95: + # must use 9.95 for proper sizing. For example, 9.99 will be + # rounded to 10.0 with the .1f format string (which is too long) + format = '%.1f%s%s' + else: + format = '%.0f%s%s' + + return(format % (number, space, symbols[depth])) + + def find_available_drpms(conduit, newpkg): """Find any applicable drpms for newpkg newpkg is a TransactionMember""" @@ -89,9 +133,12 @@ def find_available_drpms(conduit, newpkg): # don't even try to download it. try: drpm.verifySequence(sequence) - chosen_drpm = p_repo.deltalist[key] - chosen_drpm['baseurl'] = p_repo.baseurl[0] except: conduit.info(5, "Verification of %s failed" % seq) + else: + chosen_drpm = p_repo.deltalist[key] + chosen_drpm['baseurl'] = p_repo.baseurl[0] + newpkg.po.oldpkg_string = "%s.%s %s:%s-%s" % (oldpkg.name, oldpkg.arch, oldpkg.epoch, oldpkg.version, oldpkg.release) + newpkg.po.newpkg_string = "%s.%s %s:%s-%s" % (newpkg.name, newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release) return (chosen_drpm, installed, is_local, drpm_enabled) |