summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJonathan Dieter <jdieter@gmail.com>2007-03-28 17:40:25 +0300
committerJonathan Dieter <jdieter@gmail.com>2007-03-28 17:40:25 +0300
commite2c8c0c5f5ed7e2674380a37d4bd760cdc5e1289 (patch)
treeaa7d7fa694c391efeedb16dbd45caa5cc5d1ffd3
parentff77b8f02a429541f08829536fc8e4da5f38b6e4 (diff)
downloadpresto-e2c8c0c5f5ed7e2674380a37d4bd760cdc5e1289.zip
presto-e2c8c0c5f5ed7e2674380a37d4bd760cdc5e1289.tar.gz
presto-e2c8c0c5f5ed7e2674380a37d4bd760cdc5e1289.tar.xz
Bump to 0.3.0. Start downloading deltarpms ourselves.
Signed-off-by: Jonathan Dieter <jdieter@gmail.com>
-rw-r--r--ChangeLog6
-rw-r--r--Makefile1
-rwxr-xr-xmakerepo/createprestorepo.py8
-rw-r--r--presto.py104
-rw-r--r--shared/deltarpm.py2
-rw-r--r--shared/prestoDownload.py169
-rw-r--r--shared/prestoRepo.py16
-rw-r--r--shared/prestoTransaction.py18
8 files changed, 244 insertions, 80 deletions
diff --git a/ChangeLog b/ChangeLog
index 9534a55..624c2d7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,9 @@
+* Wed Mar 28 2007 Jonathan Dieter <jdieter@gmail.com> - 0.3.0
+ - Massive changes to downloading structure
+ - When unable to rebuild drpm, we now download full rpm
+ - Stop doing slow MD5 check and just check RPM header
+ while we have a prelink bug
+
* Mon Mar 26 2007 Jonathan Dieter <jdieter@gmail.com> - 0.2.9
- Fix another mirrorlist bug
- Minor optimization
diff --git a/Makefile b/Makefile
index 23f7848..a335109 100644
--- a/Makefile
+++ b/Makefile
@@ -12,4 +12,5 @@ install:
install -m 644 shared/prestomdparser.py $(DESTDIR)/usr/share/presto
install -m 644 shared/prestoTransaction.py $(DESTDIR)/usr/share/presto
install -m 644 shared/prestoLog.py $(DESTDIR)/usr/share/presto
+ install -m 644 shared/prestoDownload.py $(DESTDIR)/usr/share/presto
install -m 644 shared/deltarpm.py $(DESTDIR)/usr/share/presto
diff --git a/makerepo/createprestorepo.py b/makerepo/createprestorepo.py
index 0fee5b4..a217045 100755
--- a/makerepo/createprestorepo.py
+++ b/makerepo/createprestorepo.py
@@ -23,7 +23,7 @@ import dumpMetadata
from dumpMetadata import _gzipOpen, getChecksum
#### import Utils
-DEBUG = False
+DEBUG = True
#### Utils.setdebug(DEBUG)
SUFFIX='drpm'
@@ -138,7 +138,7 @@ def genDeltaRPM(ts, newrpm, oldrpm, is_new_package, srcdir, dstdir, locroot):
XML_oldrpm(locroot, deltaRPMName, oldrpm, newrpm, sequence, dsize)
if DEBUG:
print "DEBUG skipping %s" % (deltaRPMName)
- elif os.path.exists("%s%s.dontdelta" % (srcdir, deltaRPMName)) or os.path.getsize(f1) > 70000000:
+ elif os.path.exists("%s%s.dontdelta" % (srcdir, deltaRPMName)):
pass
else:
deltaCommand = 'makedeltarpm -s %s%s.seq %s %s %s%s' % (srcdir, deltaRPMName, f2, f1, srcdir, deltaRPMName)
@@ -208,7 +208,7 @@ def createPrestoRepo(srcdir, dstdir):
print ' Nothing found.'
return changed
assert srcfiles[0].startswith(srcdir)
-
+
# Check whether dstdir exists, and if it doesn't, create it
if not os.access(dstdir, os.F_OK):
os.makedirs(dstdir, 0755)
@@ -264,7 +264,7 @@ def createPrestoRepo(srcdir, dstdir):
# Generate delta rpm
is_new_package = True
locroot = xmlroot
- for rpm in l[1:]:
+ for rpm in l[1:]:
(is_new_package, locroot) = genDeltaRPM(ts, l[0], rpm, is_new_package, srcdir, dstdir, locroot)
if not len(srcfiles):
diff --git a/presto.py b/presto.py
index 60e391c..b270212 100644
--- a/presto.py
+++ b/presto.py
@@ -30,6 +30,7 @@ from prestoRepo import PrestoRepository
from prestomdparser import PrestoMDParser
import prestoTransaction
import prestoLog
+import prestoDownload
requires_api_version = '2.1'
LOG_FILE = "/var/log/presto.log"
@@ -37,7 +38,7 @@ plugin_type = (TYPE_INTERACTIVE,)
rpm_size = 0
drpm_size = 0
-was_drpm = False
+drpm_count = 0
# Configuration stuff
def config_hook(conduit):
@@ -75,7 +76,7 @@ def postreposetup_hook(conduit):
def postresolve_hook(conduit):
global rpm_size
global drpm_size
- global was_drpm
+ global drpm_count
opts, commands = conduit.getCmdLine()
if not opts.disablepresto:
@@ -87,31 +88,24 @@ def postresolve_hook(conduit):
# If a drpm was found, change certain package information so it reflects
# the drpm, not the rpm.
if chosen_drpm != None:
- was_drpm = True
- rpm_size += int(newpkg.po.simple['packagesize'])
+ newpkg.po.has_drpm = True
+ conduit.info(2, "Found deltarpm update for %s.%s %s:%s-%s" % (newpkg.name, newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release))
+ # In yum 3.0.x, this doesn't get defined if you run "yum update x" rather than "yum update"
+ rpm_size += int(newpkg.po.size)
drpm_size += int(chosen_drpm['size'])
- newpkg.po.hasdrpm = True
- newpkg.po.simple['basepath'] = chosen_drpm['baseurl']
- newpkg.po.simple['realpackagesize'] = newpkg.po.simple['packagesize']
+ newpkg.po.simple['realpackagesize'] = newpkg.po.size
newpkg.po.simple['packagesize'] = chosen_drpm['size']
- newpkg.po.simple['realrelativepath'] = newpkg.po.simple['relativepath']
- newpkg.po.simple['relativepath'] = chosen_drpm['drpm_filename']
- newpkg.po.reallocalpath = newpkg.po.localpath
- newpkg.po.localpath = newpkg.po.repo.deltasdir + "/" + os.path.basename(chosen_drpm['drpm_filename'])
+ newpkg.po.simple['deltasize'] = chosen_drpm['size']
+ newpkg.po.simple['deltarelativepath'] = chosen_drpm['drpm_filename']
+ newpkg.po.simple['deltachecksumtype'] = chosen_drpm['checksum_type']
+ newpkg.po.simple['deltachecksum'] = chosen_drpm['checksum']
+ newpkg.po.simple['deltalocalpath'] = newpkg.po.repo.deltasdir + "/" + os.path.basename(chosen_drpm['drpm_filename'])
newpkg.po.to = newpkg
newpkg.realpkgtup = newpkg.pkgtup
newpkg.pkgtup = (newpkg.name + " *", newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release)
- for (csum_type, csum, csumid) in newpkg.po._checksums:
- if csumid:
- newpkg.po._realchecksum = (csum_type, csum, csumid)
- newpkg.po._checksums.remove((csum_type, csum, csumid))
- csum_type = chosen_drpm['checksum_type']
- csum = chosen_drpm['checksum']
- newpkg.po._checksums.append((csum_type, csum, csumid))
-
- conduit.info(2, "Found deltarpm update for %s.%s %s:%s.%s" % (newpkg.name, newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release))
+ newpkg.po.hasdrpm = True
+ drpm_count += 1
else:
- newpkg.po.hasdrpm = False
if installed and drpm_enabled and not local:
try:
rpm_size += int(newpkg.po.simple['packagesize'])
@@ -120,59 +114,27 @@ def postresolve_hook(conduit):
pass
return
- # Free up memory used by deleting Presto repositories
- for active_repo in conduit.getRepos().listEnabled():
- if active_repo.p_repo.enabled:
- del active_repo.p_repo
-def postdownload_hook(conduit):
- global was_drpm
+def predownload_hook(conduit):
+ global drpm_count
+
+ pkglist = conduit.getDownloadPackages()
opts, commands = conduit.getCmdLine()
- if not opts.disablepresto and was_drpm:
- failure = False
- # Cycle through packages to see if we've downloaded a deltarpm
- conduit.info(2, "Rebuilding full packages from deltas")
- for pkg in conduit.getDownloadPackages():
- if pkg.hasdrpm and pkg.verifyLocalPkg():
- # Apply deltarpm and save where rpm would have been saved
- drpm = deltarpm.DeltaRpmWrapper(conduit)
- this_failure = False
- try:
- drpm.apply(pkg.reallocalpath, pkg.localpath)
- except:
- conduit.info(2, "Error rebuilding rpm from %s!" % pkg.localpath)
- failure = True
- this_failure = True
- if not this_failure:
- drpm_path = pkg.localpath
-
- # Change package information to reflect original rpm information
- pkg.to.pkgtup = pkg.to.realpkgtup
- pkg.localpath = pkg.reallocalpath
- pkg.simple['packagesize'] = pkg.simple['realpackagesize']
- del pkg.simple['basepath']
- for (csum_type, csum, csumid) in pkg._checksums:
- if csumid:
- pkg._checksums.remove((csum_type, csum, csumid))
- pkg._checksums.append(pkg._realchecksum)
-
- # Check to see whether or not we should keep the drpms
- # FIXME: Is there any way to see whether or not a Boolean option was not set?
- if conduit.confBool('main', 'neverkeepdeltas'):
- delete = True
- elif conduit.confBool('main', 'keepdeltas'):
- delete = False
- elif conduit.getConf().keepcache != 0:
- delete = False
- else:
- delete = True
-
- if delete:
- os.unlink(drpm_path)
-
- if failure:
- raise PluginYumExit("Error rebuilding at least one deltarpm. Please report this error to\nhttp://hosted.fedoraproject.org/projects/presto/newticket. To bypass this problem, run yum \nwith the --disablepresto option")
+ if not opts.disablepresto and drpm_count > 0:
+ # Download deltarpms
+ problems = prestoDownload.downloadPkgs(conduit, pkglist)
+
+ # If 'exitondownloaderror' is on, exit
+ if conduit.confBool('main', 'exitondownloaderror') and len(problems.keys()) > 0:
+ errstring = ''
+ errstring += 'Error Downloading Packages:\n'
+ for key in problems.keys():
+ errors = misc.unique(problems[key])
+ for error in errors:
+ errstring += ' %s: %s\n' % (key, error)
+ raise PluginYumExit(errstring)
+
def posttrans_hook(conduit):
global rpm_size
diff --git a/shared/deltarpm.py b/shared/deltarpm.py
index a2aea2e..710a8bb 100644
--- a/shared/deltarpm.py
+++ b/shared/deltarpm.py
@@ -80,7 +80,7 @@ class DeltaRpmWrapper:
constructs file names and paths based on given RpmDescription and instance settings for directories"""
self.conduit.info(7, '%s.verify(%s)' % (self.__class__, sequence))
p = Process(self.conduit)
- p.run(APPLY, '-c', '-s', sequence)
+ p.run(APPLY, '-s', sequence)
if p.returnCode():
# in case of error, raise exception
raise Exception("Could not verify sequence of deltarpm: %d" % (p.returnCode()))
diff --git a/shared/prestoDownload.py b/shared/prestoDownload.py
new file mode 100644
index 0000000..45318ba
--- /dev/null
+++ b/shared/prestoDownload.py
@@ -0,0 +1,169 @@
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+#
+# Copyright 2005 Duke University
+# Copyright 2007 Jonathan Dieter
+
+import os
+from yum import misc
+from yum import Errors
+from yum import types
+from urlgrabber.grabber import URLGrabError
+import deltarpm
+
+def verifyDelta(fo, po, conduit, raiseError):
+ """verifies the deltarpm is what we expect it to be
+ raiseError = defaults to 0 - if 1 then will raise
+ a URLGrabError if the file does not check out.
+ otherwise it returns false for a failure, true for success"""
+
+ if type(fo) is types.InstanceType:
+ fo = fo.filename
+
+ try:
+ verifyChecksum(fo, po.returnSimple('deltachecksumtype'), po.returnSimple('deltachecksum'))
+ except:
+ if raiseError:
+ raise URLGrabError(-1, 'Package does not match intended download')
+ else:
+ return False
+
+ return True
+
+
+def verifyChecksum(filename, checksumType, csum):
+ """Verify the checksum of the file versus the
+ provided checksum"""
+
+ try:
+
+ filesum = misc.checksum(checksumType, filename)
+ except Errors.MiscError, e:
+ raise URLGrabError(-3, 'Could not perform checksum')
+
+ if filesum != csum:
+ raise URLGrabError(-1, 'Package does not match checksum')
+
+ return 0
+
+
+def downloadPkgs(conduit, pkglist):
+ """download list of package objects handed to you, return errors"""
+
+ opts, commands = conduit.getCmdLine()
+
+ errors = {}
+ def adderror(po, msg):
+ errors.setdefault(po, []).append(msg)
+
+ # Check whether drpm is already downloaded
+ repo_cached = False
+ remote_pkgs = []
+ rebuild_pkgs = []
+ for po in conduit.getDownloadPackages():
+ if hasattr(po, 'has_drpm') and po.has_drpm:
+ po.to.pkgtup = po.to.realpkgtup
+ local = po.returnSimple('deltalocalpath')
+ if os.path.exists(local):
+ cursize = os.stat(local)[6]
+ totsize = long(po.returnSimple('deltasize'))
+ try:
+ verifyChecksum(local, po.returnSimple('deltachecksumtype'), po.returnSimple('deltachecksum'))
+ except:
+ if po.repo.p_repo.cache:
+ repo_cached = True
+ adderror(po, 'package fails checksum but caching is '
+ 'enabled for %s' % po.repo.p_repo.id)
+
+ if cursize >= totsize: # otherwise keep it around for regetting
+ os.unlink(local)
+ else:
+ # Deltarpm is local and good, let's put it in the rebuild list
+ conduit.info(5, "using local copy of deltarpm for %s" % po)
+ rebuild_pkgs.append(po)
+ continue
+ remote_pkgs.append(po)
+
+ # Download deltarpms
+ i = 0
+ for po in remote_pkgs:
+ i += 1
+ checkfunc = (verifyDelta, (po, conduit, 1), {})
+ cache = po.repo.p_repo.http_caching != 'none'
+ dirstat = os.statvfs(po.repo.deltasdir)
+ if (dirstat.f_bavail * dirstat.f_bsize) <= long(po.size):
+ adderror(po, 'Insufficient space in download directory %s '
+ 'to download' % po.repo.deltasdir)
+ continue
+ po.simple['reallocalpath'] = po.localpath
+ po.localpath = po.returnSimple('deltalocalpath')
+ po.simple['realrelativepath'] = po.returnSimple('relativepath')
+ po.simple['relativepath'] = po.returnSimple('deltarelativepath')
+ try:
+ text = '(%s/%s): %s' % (i, len(remote_pkgs), os.path.basename(po.returnSimple('deltarelativepath')))
+ deltalocal = po.repo.p_repo.getPackage(po, checkfunc=checkfunc, text=text, cache=cache)
+ except Errors.RepoError, e:
+ adderror(po, str(e))
+ else:
+ rebuild_pkgs.append(po)
+
+ if errors.has_key(po):
+ del errors[po]
+
+ po.simple['relativepath'] = po.returnSimple('realrelativepath')
+ po.localpath = po.returnSimple('reallocalpath')
+ if po.simple.has_key('realpackagesize'):
+ po.simple['packagesize'] = po.returnSimple('realpackagesize')
+ del po.simple['realpackagesize']
+ del po.simple['realrelativepath']
+ del po.simple['reallocalpath']
+ po.simple['deltalocalpath'] = deltalocal
+
+ # Rebuild rpms from downloaded deltarpms
+ for po in rebuild_pkgs:
+ deltalocal = po.returnSimple('deltalocalpath')
+ drpm = deltarpm.DeltaRpmWrapper(conduit)
+ try:
+ conduit.info(2, "Building %s from %s" % (os.path.basename(po.localpath), os.path.basename(deltalocal)))
+ drpm.apply(po.localpath, deltalocal)
+ except:
+ conduit.info(2, "Error rebuilding rpm from %s! Will download full package." % os.path.basename(deltalocal))
+ try:
+ os.unlink(po.localpath)
+ except:
+ pass
+ else:
+ # Set package type to local, so yum doesn't try to download it later
+ po.pkgtype = "local"
+
+ # Check to see whether or not we should keep the drpms
+ # FIXME: Is there any way to see whether or not a Boolean option was not set?
+ if conduit.confBool('main', 'neverkeepdeltas'):
+ delete = True
+ elif conduit.confBool('main', 'keepdeltas'):
+ delete = False
+ elif conduit.getConf().keepcache != 0:
+ delete = False
+ else:
+ delete = True
+
+ if delete:
+ try:
+ os.unlink(deltalocal)
+ except:
+ pass
+
+ return errors
+
+
diff --git a/shared/prestoRepo.py b/shared/prestoRepo.py
index f9c4648..582dc2f 100644
--- a/shared/prestoRepo.py
+++ b/shared/prestoRepo.py
@@ -158,11 +158,11 @@ class PrestoRepository(Repository):
def dump(self):
output = '[%s]\n' % self.id
- vars = ['name', 'bandwidth', 'enabled',
+ vars = ['id', 'bandwidth', 'enabled',
'keepalive', 'proxy',
'proxy_password', 'proxy_username',
'retries', 'throttle', 'timeout', 'mirrorlist',
- 'cachedir', 'deltasdir' ]
+ 'cachedir' ]
vars.sort()
for attr in vars:
output = output + '%s = %s\n' % (attr, getattr(self, attr))
@@ -367,6 +367,18 @@ class PrestoRepository(Repository):
return result
+ def getPackage(self, package, checkfunc = None, text = None, cache = True):
+ remote = package.returnSimple('relativepath')
+ local = package.localPkg()
+ basepath = package.returnSimple('basepath')
+
+ return self.__get(url=basepath,
+ relative=remote,
+ local=local,
+ checkfunc=checkfunc,
+ text=text,
+ cache=cache
+ )
def metadataCurrent(self):
"""Check if there is a metadata_cookie and check its age. If the
diff --git a/shared/prestoTransaction.py b/shared/prestoTransaction.py
index 71b90f3..3d387a4 100644
--- a/shared/prestoTransaction.py
+++ b/shared/prestoTransaction.py
@@ -24,15 +24,29 @@ def find_available_drpms(conduit, newpkg):
rpmdb = conduit.getRpmDB()
+ is_local = False
+
# Set p_repo to be packages delta repository or set to False if
# there is no delta repository
try:
p_repo = newpkg.po.repo.p_repo
drpm_enabled = p_repo.enabled
- if os.path.exists(newpkg.po.localpath):
+
+ po = newpkg.po
+ if hasattr(po, 'pkgtype') and po.pkgtype == 'local':
is_local = True
else:
- is_local = False
+ local = po.localPkg()
+ if os.path.exists(local):
+ cursize = os.stat(local)[6]
+ totsize = long(po.size)
+ if not po.verifyLocalPkg():
+ if cursize >= totsize: # otherwise keep it around for regetting
+ os.unlink(local)
+ else:
+ conduit.info(5, "using local copy of %s" % po)
+ is_local = True
+
except:
conduit.info(5, "No Presto repository information for %s.%s %i:%s-%s" % (newpkg.name, newpkg.arch, int(newpkg.epoch), newpkg.version, newpkg.release))
drpm_enabled = False