summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJonathan Dieter <jdieter@gmail.com>2007-04-04 17:35:37 +0300
committerJonathan Dieter <jdieter@gmail.com>2007-04-04 17:35:37 +0300
commit1942f8d44c679fc612c97202727ce362c3acb130 (patch)
tree4af5637766d4e44e1b4bc451861a9281f839e123
parentbdfb4037667c6af7d0537f9bc36b33d55d86e059 (diff)
downloadpresto-1942f8d44c679fc612c97202727ce362c3acb130.zip
presto-1942f8d44c679fc612c97202727ce362c3acb130.tar.gz
presto-1942f8d44c679fc612c97202727ce362c3acb130.tar.xz
Add improved logging. Many bug fixes.
Signed-off-by: Jonathan Dieter <jdieter@gmail.com>
-rw-r--r--yum-presto/ChangeLog4
-rw-r--r--yum-presto/presto.py32
-rw-r--r--yum-presto/shared/prestoDownload.py15
-rw-r--r--yum-presto/shared/prestoLog.py113
-rw-r--r--yum-presto/shared/prestoThread.py3
-rw-r--r--yum-presto/shared/prestoTransaction.py51
6 files changed, 155 insertions, 63 deletions
diff --git a/yum-presto/ChangeLog b/yum-presto/ChangeLog
index 4612d88..4b6b95f 100644
--- a/yum-presto/ChangeLog
+++ b/yum-presto/ChangeLog
@@ -1,3 +1,7 @@
+* Wed Apr 4 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.5
+ - Many small bug fixes
+ - Improved logging - Large changes
+
* Tue Apr 3 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.4
- Add patch from Ahmed Kamal to put rebuilding of rpms into a
different thread
diff --git a/yum-presto/presto.py b/yum-presto/presto.py
index e9908ee..a5bd77e 100644
--- a/yum-presto/presto.py
+++ b/yum-presto/presto.py
@@ -17,6 +17,7 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# Copyright 2005 Duke University
+# Copyright 2007 Jonathan Dieter
from yum.plugins import TYPE_INTERACTIVE, PluginYumExit
from yum import config
@@ -39,9 +40,12 @@ plugin_type = (TYPE_INTERACTIVE,)
rpm_size = 0
drpm_size = 0
drpm_count = 0
+log = None
# Configuration stuff
-def config_hook(conduit):
+def config_hook(conduit):
+ global log
+
# Set up repository specific deltarpm url and mirrorlist
config.RepoConf.deltaurl = config.UrlListOption()
config.RepoConf.deltamirrorlist = config.UrlOption()
@@ -51,6 +55,9 @@ def config_hook(conduit):
parser.add_option('', '--disablepresto', dest='disablepresto',
action='store_true', default=False,
help="disable Presto plugin and don't download any deltarpms")
+
+ # Set up logging
+ log = prestoLog.PrestoLog(conduit, LOG_FILE)
# Set up Presto repositories
def postreposetup_hook(conduit):
@@ -117,33 +124,40 @@ def postresolve_hook(conduit):
def predownload_hook(conduit):
global drpm_count
+ global log
pkglist = conduit.getDownloadPackages()
opts, commands = conduit.getCmdLine()
if not opts.disablepresto and drpm_count > 0:
+ conduit.info(2, "Downloading DeltaRPMs:")
+
# Download deltarpms
- problems = prestoDownload.downloadPkgs(conduit, pkglist)
+ problems = prestoDownload.downloadPkgs(conduit, pkglist, log)
# If 'exitondownloaderror' is on, exit
if conduit.confBool('main', 'exitondownloaderror') and len(problems.keys()) > 0:
- errstring = ''
- errstring += 'Error Downloading Packages:\n'
+ errstring = 'Error Downloading Packages:\n'
for key in problems.keys():
errors = misc.unique(problems[key])
for error in errors:
errstring += ' %s: %s\n' % (key, error)
raise PluginYumExit(errstring)
+ else:
+ conduit.info(2, "Downloading RPMs:")
def posttrans_hook(conduit):
global rpm_size
global drpm_size
- global LOG_FILE
+ global log
+
+ log.close()
if rpm_size > 0:
- prestoLog.log(conduit, LOG_FILE, rpm_size, drpm_size)
-
- conduit.info(2, "Size of all updates downloaded from Presto-enabled repositories: %i bytes" % drpm_size)
- conduit.info(2, "Size of updates that would have been downloaded if Presto wasn't enabled: %i bytes" % rpm_size)
+ drpm_string = prestoTransaction.format_number(drpm_size)
+ rpm_string = prestoTransaction.format_number(rpm_size)
+
+ conduit.info(2, "Size of all updates downloaded from Presto-enabled repositories: %s" % drpm_string)
+ conduit.info(2, "Size of updates that would have been downloaded if Presto wasn't enabled: %s" % rpm_string)
conduit.info(2, "This is a savings of %i percent" % (100 - ((drpm_size * 100) / rpm_size)))
diff --git a/yum-presto/shared/prestoDownload.py b/yum-presto/shared/prestoDownload.py
index f21d409..30e7514 100644
--- a/yum-presto/shared/prestoDownload.py
+++ b/yum-presto/shared/prestoDownload.py
@@ -60,7 +60,7 @@ def verifyChecksum(filename, checksumType, csum):
return 0
-def reconstruct(conduit, po):
+def reconstruct(conduit, po, log):
deltalocal = po.returnSimple('deltalocalpath')
retlist = ""
@@ -75,7 +75,10 @@ def reconstruct(conduit, po):
except:
pass
else:
- retlist += "Built %s from deltarpm\n" % (os.path.basename(po.localpath), os.path.basename(deltalocal))
+ # log success
+ log.log(po.oldpkg_string, po.newpkg_string, os.stat(po.localpath)[6], os.stat(deltalocal)[6])
+
+ #retlist += "Built %s from deltarpm\n" % os.path.basename(po.localpath)
# Check to see whether or not we should keep the drpms
# FIXME: Is there any way to see whether or not a Boolean option was not set?
if conduit.confBool('main', 'neverkeepdeltas'):
@@ -96,7 +99,7 @@ def reconstruct(conduit, po):
-def downloadPkgs(conduit, pkglist):
+def downloadPkgs(conduit, pkglist, log):
"""download list of package objects handed to you, return errors"""
opts, commands = conduit.getCmdLine()
@@ -134,7 +137,7 @@ def downloadPkgs(conduit, pkglist):
else:
# Deltarpm is local and good, let's put it in the rebuild thread.
conduit.info(5, "using local copy of deltarpm for %s" % po)
- queue.put((conduit, po))
+ queue.put((conduit, po, log))
continue
remote_pkgs.append(po)
@@ -159,7 +162,7 @@ def downloadPkgs(conduit, pkglist):
except Errors.RepoError, e:
adderror(po, str(e))
else:
- queue.put((conduit, po))
+ queue.put((conduit, po, log))
po.simple['deltalocalpath'] = deltalocal
if errors.has_key(po):
@@ -180,6 +183,8 @@ def downloadPkgs(conduit, pkglist):
curthread.messages = ""
lock.release()
+ conduit.info(2, "Rebuilding rpms from deltarpms")
+
# Tell build thread that there are no more drpms and wait for it to exit
curthread.can_exit = True
curthread.join()
diff --git a/yum-presto/shared/prestoLog.py b/yum-presto/shared/prestoLog.py
index 3c0c1e6..1323346 100644
--- a/yum-presto/shared/prestoLog.py
+++ b/yum-presto/shared/prestoLog.py
@@ -15,57 +15,78 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# Copyright 2005 Duke University
-def log(conduit, LOG_FILE, rpm_size, drpm_size):
- # Open log file for reading
- try:
- log_file = open(LOG_FILE, "r")
- log_exists = True
- except:
- conduit.info(5, "Info: %s doesn't exist. Will create." % LOG_FILE)
- log_exists = False
-
- # Log file doesn't exist, create
- if not log_exists:
+class PrestoLog:
+ def __init__(self, conduit, log_filename):
+ # Open log file for reading
try:
- log_file = open(LOG_FILE, "w")
- log_file.write("Download Size (without DRPM),Download Size (with DRPM),Percentage Savings,Total Percentage Savings\n")
- log_file.close()
+ log_file = open(log_filename, "r")
log_exists = True
except:
- conduit.info(2, "Warning: Unable to write to %s" % LOG_FILE)
+ conduit.info(7, "Info: %s doesn't exist. Will create." % log_filename)
+ log_exists = False
+
+ # Log file doesn't exist, create
+ if not log_exists:
+ try:
+ log_file = open(log_filename, "w")
+ log_file.close()
+ log_exists = True
+ except:
+ conduit.info(2, "Warning: Unable to write to %s" % log_filename)
+ if log_exists:
+ try:
+ log_file = open(log_filename, "r")
+ except:
+ conduit.info(2, "Warning: Unable to open %s for reading." % log_filename)
+ log_exists = False
+
+ # Cycle through items already in log so we can come up with total savings
if log_exists:
+ self.total_rpm_size = 0
+ self.total_drpm_size = 0
+
+ # Get rid of header line
+ log_file.readline()
+
+ data = log_file.readline()
+ while data != "":
+ fc = data.rfind("-")
+ sc = data.rfind("-", 0, fc-1)
+ tc = data.rfind("-", 0, sc-1)
+ lc = data.rfind("-", 0, tc-1)
+ if lc != -1 and tc != -1 and sc != -1 and fc != -1:
+ self.total_rpm_size += int(data[lc+1:tc])
+ self.total_drpm_size += int(data[tc+1:sc])
+ data = log_file.readline()
+ log_file.close()
+
try:
- log_file = open(LOG_FILE, "r")
+ log_file = open(log_filename, "a")
except:
- conduit.info(2, "Warning: Unable to open %s for reading." % LOG_FILE)
- log_exists = False
+ conduit.info(2, "Warning: Unable to open %s for writing." % log_filename)
+ self.log_file = None
+ else:
+ self.log_file = log_filename
+ log_file.close()
+
+ def log(self, oldrpm_name, newrpm_name, rpm_size, drpm_size):
+ # Write data to log
+ self.total_rpm_size += rpm_size
+ self.total_drpm_size += drpm_size
+ if self.log_file != None:
+ try:
+ log_file = open(self.log_file, "a")
+ except:
+ pass
+ else:
+ log_file.write("%s => %s - %i - %i - %i - %i\n" % (oldrpm_name, newrpm_name, rpm_size, drpm_size, 100 - ((drpm_size * 100) / rpm_size), 100 - ((self.total_drpm_size * 100) / self.total_rpm_size)))
+ log_file.close()
+
- # Cycle through items already in log so we can come up with total savings
- if log_exists:
- total_rpm_size = 0
- total_drpm_size = 0
-
- # Get rid of header line
- log_file.readline()
-
- data = log_file.readline()
- while data != "":
- fc = data.find(",")
- sc = data.find(",", fc + 1)
- total_rpm_size += int(data[:fc])
- total_drpm_size += int(data[fc + 1:sc])
- data = log_file.readline()
- log_file.close()
- total_rpm_size += rpm_size
- total_drpm_size += drpm_size
+ def close(self):
+ if self.log_file != None:
+ try:
+ self.log_file.close()
+ except:
+ pass
- try:
- log_file = open(LOG_FILE, "a")
- except:
- conduit.info(2, "Warning: Unable to open %s for writing." % LOG_FILE)
- log_exists = False
-
- # Write data to log
- if log_exists:
- log_file.write("%i,%i,%i,%i\n" % (rpm_size, drpm_size, 100 - ((drpm_size * 100) / rpm_size), 100 - ((total_drpm_size * 100) / total_rpm_size)))
- log_file.close()
diff --git a/yum-presto/shared/prestoThread.py b/yum-presto/shared/prestoThread.py
index e67db76..e090910 100644
--- a/yum-presto/shared/prestoThread.py
+++ b/yum-presto/shared/prestoThread.py
@@ -44,7 +44,8 @@ class ReconstructionThread(threading.Thread):
if self.messages != "":
conduit.info(2, self.messages[:-1])
self.messages = ""
- conduit.info(2, messages[:-1])
+ if messages != "":
+ conduit.info(2, messages[:-1])
else:
# We may be downloading drpms still, so queue messages
self.lock.acquire()
diff --git a/yum-presto/shared/prestoTransaction.py b/yum-presto/shared/prestoTransaction.py
index 166fe6d..4f08391 100644
--- a/yum-presto/shared/prestoTransaction.py
+++ b/yum-presto/shared/prestoTransaction.py
@@ -1,5 +1,7 @@
# author: Jonathan Dieter <jdieter@gmail.com>
#
+# format_number taken almost completely from progress_meter.py in yum
+#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
@@ -14,10 +16,52 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# Copyright 2005 Duke University
+# Copyright 2007 Jonathan Dieter
import os
import deltarpm
+def format_number(number, SI=False, space=''):
+ """Turn numbers into human-readable metric-like numbers"""
+ symbols = ['', # (none)
+ 'K', # kilo
+ 'M', # mega
+ 'G', # giga
+ 'T', # tera
+ 'P', # peta
+ 'E', # exa
+ 'Z', # zetta
+ 'Y'] # yotta
+
+ if SI: step = 1000.0
+ else: step = 1024.0
+
+ thresh = 999
+ depth = 0
+
+ # we want numbers between
+ while number > thresh:
+ depth = depth + 1
+ number = number / step
+
+ # just in case someone needs more than 1000 yottabytes!
+ diff = depth - len(symbols) + 1
+ if diff > 0:
+ depth = depth - diff
+ number = number * thresh**depth
+
+ if type(number) == type(1) or type(number) == type(1L):
+ format = '%i%s%s'
+ elif number < 9.95:
+ # must use 9.95 for proper sizing. For example, 9.99 will be
+ # rounded to 10.0 with the .1f format string (which is too long)
+ format = '%.1f%s%s'
+ else:
+ format = '%.0f%s%s'
+
+ return(format % (number, space, symbols[depth]))
+
+
def find_available_drpms(conduit, newpkg):
"""Find any applicable drpms for newpkg
newpkg is a TransactionMember"""
@@ -89,9 +133,12 @@ def find_available_drpms(conduit, newpkg):
# don't even try to download it.
try:
drpm.verifySequence(sequence)
- chosen_drpm = p_repo.deltalist[key]
- chosen_drpm['baseurl'] = p_repo.baseurl[0]
except:
conduit.info(5, "Verification of %s failed" % seq)
+ else:
+ chosen_drpm = p_repo.deltalist[key]
+ chosen_drpm['baseurl'] = p_repo.baseurl[0]
+ newpkg.po.oldpkg_string = "%s.%s %s:%s-%s" % (oldpkg.name, oldpkg.arch, oldpkg.epoch, oldpkg.version, oldpkg.release)
+ newpkg.po.newpkg_string = "%s.%s %s:%s-%s" % (newpkg.name, newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release)
return (chosen_drpm, installed, is_local, drpm_enabled)