diff options
Diffstat (limited to 'yum-presto-legacy/shared')
| -rw-r--r-- | yum-presto-legacy/shared/deltarpm.py | 86 | ||||
| -rw-r--r-- | yum-presto-legacy/shared/prestoDownload.py | 188 | ||||
| -rw-r--r-- | yum-presto-legacy/shared/prestoLog.py | 92 | ||||
| -rw-r--r-- | yum-presto-legacy/shared/prestoRepo.py | 615 | ||||
| -rw-r--r-- | yum-presto-legacy/shared/prestoThread.py | 53 | ||||
| -rw-r--r-- | yum-presto-legacy/shared/prestoTransaction.py | 140 | ||||
| -rw-r--r-- | yum-presto-legacy/shared/prestomdparser.py | 169 |
7 files changed, 1343 insertions, 0 deletions
diff --git a/yum-presto-legacy/shared/deltarpm.py b/yum-presto-legacy/shared/deltarpm.py new file mode 100644 index 0000000..97a3cc6 --- /dev/null +++ b/yum-presto-legacy/shared/deltarpm.py @@ -0,0 +1,86 @@ +# author: Jonathan Dieter <jdieter@gmail.com> +# +# mostly taken from deltarpm.py created by +# Lars Herrmann <herrmann@redhat.com> +# and modified for Presto by +# Ahmed Kamal <email.ahmedkamal@googlemail.com> +# +# license: GPL (see COPYING file in distribution) +# +# this module provides a python wrapper around deltarpm tools written by suse +# +# TODO: catch exceptions wherever possible and raise useful ones ;) +# see TODO lines in methods + +APPLY='/usr/bin/applydeltarpm' + +import popen2 +import string +import os + +class Process: + """wrapper class to execute programs and return exitcode and output (stdout and stderr combined)""" + def __init__(self, conduit): + self.__stdout=None + self.__returncode=None + self.__command=None + self.__args=None + self.conduit = conduit + + def run(self, command, *args): + self.__command=command + self.__args=args + cmdline=command+" "+string.join(args, " ") + self.conduit.info(7, '%s.%s: executing %s' % (self.__class__, 'run', cmdline)) + pipe = popen2.Popen4(cmdline) + self.__stdout=pipe.fromchild.read() + retcode = pipe.wait() + if os.WIFEXITED(retcode): + self.__returncode = os.WEXITSTATUS(retcode) + else: + self.__returncode = retcode + # fallback to old implementation - works better ? + #stdoutp = os.popen(cmdline,'r',1) + #self.__stdout = stdoutp.read() + #retcode = stdoutp.close() + #if retcode is None: + # self.__returncode = 0 + #else: + # self.__returncode = retcode + + def getOutput(self): + return self.__stdout + + def returnCode(self): + return self.__returncode + +class DeltaRpmWrapper: + """wrapper around deltarpm binaries - implement methods for applying and verifying delta rpms + - raises exceptions if exitcode of binaries was != 0""" + + def __init__(self, conduit): + self.conduit = conduit + self.conduit.info(7, '%s.%s: created' % (self.__class__, '__init__')) + + def apply(self, arch, newrpmfile, deltarpmfile): + """wraps execution of applydeltarpm [-r oldrpm] deltarpm newrpm - + constructs file names and paths based on given RpmDescription and instance settings for directories""" + # TODO: test args for type == instance and __class__ == RpmDescription + self.conduit.info(7, '%s.apply(%s,%s)' % (self.__class__, newrpmfile, deltarpmfile)) + p=Process(self.conduit) + # targetrpm filename + p.run(APPLY, '-a', arch, deltarpmfile, newrpmfile) + if p.returnCode(): + # in case of error, raise exception + raise Exception("Could not apply deltarpm: %d" % (p.returnCode())) + return newrpmfile + + def verifySequence(self, arch, sequence): + """wraps execution of applydeltarpm [-r oldrpm] -s seqfilecontent - + constructs file names and paths based on given RpmDescription and instance settings for directories""" + self.conduit.info(7, '%s.verify(%s)' % (self.__class__, sequence)) + p = Process(self.conduit) + p.run(APPLY, '-a', arch, '-C', '-s', sequence) + if p.returnCode(): + # in case of error, raise exception + raise Exception("Could not verify sequence of deltarpm: %d" % (p.returnCode())) diff --git a/yum-presto-legacy/shared/prestoDownload.py b/yum-presto-legacy/shared/prestoDownload.py new file mode 100644 index 0000000..1c29608 --- /dev/null +++ b/yum-presto-legacy/shared/prestoDownload.py @@ -0,0 +1,188 @@ +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# +# Copyright 2005 Duke University +# Copyright 2007 Jonathan Dieter + +import os +from yum import misc +from yum import Errors +from yum import types +from urlgrabber.grabber import URLGrabError +import deltarpm +import prestoThread +import Queue +import thread + +def verifyDelta(fo, po, conduit, raiseError): + """verifies the deltarpm is what we expect it to be + raiseError = defaults to 0 - if 1 then will raise + a URLGrabError if the file does not check out. + otherwise it returns false for a failure, true for success""" + + if type(fo) is types.InstanceType: + fo = fo.filename + + try: + verifyChecksum(fo, po.deltachecksumtype, po.deltachecksum) + except: + if raiseError: + raise URLGrabError(-1, 'Package does not match intended download') + else: + return False + + return True + + +def verifyChecksum(filename, checksumType, csum): + """Verify the checksum of the file versus the + provided checksum""" + + try: + + filesum = misc.checksum(checksumType, filename) + except Errors.MiscError, e: + raise URLGrabError(-3, 'Could not perform checksum') + + if filesum != csum: + raise URLGrabError(-1, 'Package does not match checksum') + + return 0 + +def reconstruct(conduit, po, log): + deltalocal = po.deltalocalpath + retlist = "" + + drpm = deltarpm.DeltaRpmWrapper(conduit) + + try: + drpm.apply(po.arch, po.localpath, deltalocal) + except: + retlist += "Error rebuilding rpm from %s! Will download full package.\n" % os.path.basename(deltalocal) + try: + os.unlink(po.localpath) + except: + pass + else: + # log success + log.log(po.oldpkg_string, po.newpkg_string, os.stat(po.localpath)[6], os.stat(deltalocal)[6]) + + #retlist += "Built %s from deltarpm\n" % os.path.basename(po.localpath) + # Check to see whether or not we should keep the drpms + # FIXME: Is there any way to see whether or not a Boolean option was not set? + if conduit.confBool('main', 'neverkeepdeltas'): + delete = True + elif conduit.confBool('main', 'keepdeltas'): + delete = False + elif conduit.getConf().keepcache != 0: + delete = False + else: + delete = True + + if delete: + try: + os.unlink(deltalocal) + except: + pass + return retlist + + + +def downloadPkgs(conduit, pkglist, log): + """download list of package objects handed to you, return errors""" + + opts, commands = conduit.getCmdLine() + + errors = {} + def adderror(po, msg): + errors.setdefault(po, []).append(msg) + + # Set up thread for applying drpms + queue = Queue.Queue(0) + lock = thread.allocate_lock() + curthread = prestoThread.ReconstructionThread(queue, lock, reconstruct) + curthread.start() + + # Check whether drpm is already downloaded + repo_cached = False + remote_pkgs = [] + for po in conduit.getDownloadPackages(): + if hasattr(po, 'has_drpm') and po.has_drpm: + local = po.deltalocalpath + if os.path.exists(local): + cursize = os.stat(local)[6] + totsize = long(po.deltasize) + try: + verifyChecksum(local, po.deltachecksumtype, po.deltachecksum) + except: + if po.repo.p_repo.cache: + repo_cached = True + adderror(po, 'package fails checksum but caching is ' + 'enabled for %s' % po.repo.p_repo.id) + + if cursize >= totsize: # otherwise keep it around for regetting + os.unlink(local) + else: + # Deltarpm is local and good, let's put it in the rebuild thread. + conduit.info(5, "using local copy of deltarpm for %s" % po) + queue.put((conduit, po, log)) + continue + remote_pkgs.append(po) + + # Download deltarpms and send them to another thread to be rebuilt + i = 0 + for po in remote_pkgs: + i += 1 + checkfunc = (verifyDelta, (po, conduit, 1), {}) + cache = po.repo.p_repo.http_caching != 'none' + dirstat = os.statvfs(po.repo.deltasdir) + if (dirstat.f_bavail * dirstat.f_bsize) <= long(po.size): + adderror(po, 'Insufficient space in download directory %s ' + 'to download' % po.repo.deltasdir) + continue + try: + text = '(%s/%s): %s' % (i, len(remote_pkgs), os.path.basename(po.deltarelativepath)) + deltalocal = po.repo.p_repo.getPackage(po, checkfunc=checkfunc, text=text, cache=cache) + except Errors.RepoError, e: + adderror(po, str(e)) + else: + queue.put((conduit, po, log)) + + if errors.has_key(po): + del errors[po] + + if hasattr(po, 'realpackagesize'): + po.packagesize = po.realpackagesize + del po.realpackagesize + + # Check for waiting messages from building thread + lock.acquire() + if curthread.messages != "": + conduit.info(2, curthread.messages[:-1]) + curthread.messages = "" + lock.release() + + conduit.info(2, "Rebuilding rpms from deltarpms") + + # Tell build thread that there are no more drpms and wait for it to exit + curthread.can_exit = True + queue.put(None) + curthread.join() + + if curthread.messages != "": + conduit.info(2, curthread.messages[:-1]) + curthread.messages = "" + + return errors + diff --git a/yum-presto-legacy/shared/prestoLog.py b/yum-presto-legacy/shared/prestoLog.py new file mode 100644 index 0000000..1323346 --- /dev/null +++ b/yum-presto-legacy/shared/prestoLog.py @@ -0,0 +1,92 @@ +# author: Jonathan Dieter <jdieter@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2005 Duke University + +class PrestoLog: + def __init__(self, conduit, log_filename): + # Open log file for reading + try: + log_file = open(log_filename, "r") + log_exists = True + except: + conduit.info(7, "Info: %s doesn't exist. Will create." % log_filename) + log_exists = False + + # Log file doesn't exist, create + if not log_exists: + try: + log_file = open(log_filename, "w") + log_file.close() + log_exists = True + except: + conduit.info(2, "Warning: Unable to write to %s" % log_filename) + if log_exists: + try: + log_file = open(log_filename, "r") + except: + conduit.info(2, "Warning: Unable to open %s for reading." % log_filename) + log_exists = False + + # Cycle through items already in log so we can come up with total savings + if log_exists: + self.total_rpm_size = 0 + self.total_drpm_size = 0 + + # Get rid of header line + log_file.readline() + + data = log_file.readline() + while data != "": + fc = data.rfind("-") + sc = data.rfind("-", 0, fc-1) + tc = data.rfind("-", 0, sc-1) + lc = data.rfind("-", 0, tc-1) + if lc != -1 and tc != -1 and sc != -1 and fc != -1: + self.total_rpm_size += int(data[lc+1:tc]) + self.total_drpm_size += int(data[tc+1:sc]) + data = log_file.readline() + log_file.close() + + try: + log_file = open(log_filename, "a") + except: + conduit.info(2, "Warning: Unable to open %s for writing." % log_filename) + self.log_file = None + else: + self.log_file = log_filename + log_file.close() + + def log(self, oldrpm_name, newrpm_name, rpm_size, drpm_size): + # Write data to log + self.total_rpm_size += rpm_size + self.total_drpm_size += drpm_size + if self.log_file != None: + try: + log_file = open(self.log_file, "a") + except: + pass + else: + log_file.write("%s => %s - %i - %i - %i - %i\n" % (oldrpm_name, newrpm_name, rpm_size, drpm_size, 100 - ((drpm_size * 100) / rpm_size), 100 - ((self.total_drpm_size * 100) / self.total_rpm_size))) + log_file.close() + + + def close(self): + if self.log_file != None: + try: + self.log_file.close() + except: + pass + diff --git a/yum-presto-legacy/shared/prestoRepo.py b/yum-presto-legacy/shared/prestoRepo.py new file mode 100644 index 0000000..bc1b188 --- /dev/null +++ b/yum-presto-legacy/shared/prestoRepo.py @@ -0,0 +1,615 @@ +# author: Jonathan Dieter <jdieter@gmail.com> +# +# mostly taken from yumRepo.py (part of yum) with a few minor modifications +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2005 Duke University + +import os +import re +import time +import types +import urlparse + +from yum import Errors +from urlgrabber.grabber import URLGrabber +import urlgrabber.mirror +from urlgrabber.grabber import URLGrabError +from yum.repos import Repository +from yum import repoMDObject +from yum import parser +from yum import config +from yum import misc + +class PrestoRepository(Repository): + """ + This is an actual repository object + + Configuration attributes are pulled in from config.RepoConf. + """ + + def __init__(self, repo, conduit): + Repository.__init__(self, repo.id) + + # If there's a specific deltarpm url, use that + is_different = False + if conduit.confString(repo.id, 'deltaurl'): + self.baseurl = [conduit.confString(repo.id, 'deltaurl')] + is_different = True + conduit.info(5, 'Manual url set from presto.conf: %s' % self.baseurl) + elif repo.deltaurl != [] and repo.deltaurl != None: + self.baseurl = repo.deltaurl + is_different = True + conduit.info(5, 'Manual url set from repository conf file: %s' % self.baseurl) + else: + self.baseurl = repo.baseurl + + # If there's a specific mirrorlist, use that + if conduit.confString(repo.id, 'deltamirrorlist'): + self.mirrorlist = conduit.confString(repo.id, 'deltamirrorlist') + self.baseurl = None + is_different = True + conduit.info(5, 'Manual mirrorlist set from presto.conf: %s' % self.mirrorlist) + elif repo.deltamirrorlist != None: + self.mirrorlist = repo.deltamirrorlist + self.baseurl = None + is_different = True + conduit.info(5, 'Manual mirrorlist set from repository conf file: %s' % self.mirrorlist) + else: + if self.baseurl == repo.baseurl: + self.mirrorlist = repo.mirrorlist + else: + self.mirrorlist = None + + self.conduit = conduit + self.urls = [] + self.is_different = is_different + if is_different: + self.repoMDFile = 'repodata/prestomd.xml' + self.metadata_cookie_fn = 'presto_cachecookie' + else: + self.repoMDFile = 'repodata/repomd.xml' + self.metadata_cookie_fn = 'cachecookie' + self.repoXML = None + self.cache = 0 + self.mirrorlistparsed = 0 + self.yumvar = {} # empty dict of yumvariables for $string replacement + self._proxy_dict = {} + self.http_headers = {} + + # throw in some stubs for things that will be set by the config class + self.basecachedir = "" + self.cachedir = "" + self.pkgdir = "" + self.hdrdir = "" + self.enabled = True + + # holder for stuff we've grabbed + self.retrieved = { 'deltas':0 } + + # callbacks + self.keepalive = repo.keepalive + self.bandwidth = repo.bandwidth + self.retries = repo.retries + self.throttle = repo.throttle + self.proxy = repo.proxy + self.proxy_username = repo.proxy_username + self.proxy_password = repo.proxy_password + self.timeout = repo.timeout + self.http_caching = repo.http_caching + self.failovermethod = repo.failovermethod + self.metadata_expire = repo.metadata_expire + self.basecachedir = repo.basecachedir + self.callback = repo.callback + self.failure_obj = repo.failure_obj + self.mirror_failure_obj = repo.mirror_failure_obj + self.interrupt_callback = repo.interrupt_callback + self.drpm_list = {} + self.parent = repo + repo.p_repo = self + + + def __getProxyDict(self): + self.doProxyDict() + if self._proxy_dict: + return self._proxy_dict + return None + + # consistent access to how proxy information should look (and ensuring + # that it's actually determined for the repo) + proxy_dict = property(__getProxyDict) + + def ready(self): + """Returns true if this repository is setup and ready for use.""" + return self.repoXML is not None + + def __cmp__(self, other): + if self.id > other.id: + return 1 + elif self.id < other.id: + return -1 + else: + return 0 + + def __str__(self): + return self.id + + def _checksum(self, sumtype, file, CHUNK=2**16): + """takes filename, hand back Checksum of it + sumtype = md5 or sha + filename = /path/to/file + CHUNK=65536 by default""" + try: + return misc.checksum(sumtype, file, CHUNK) + except (Errors.MiscError, EnvironmentError), e: + raise Errors.RepoError, 'Error opening file for checksum: %s' % e + + def dump(self): + output = '[%s]\n' % self.id + vars = ['id', 'bandwidth', 'enabled', + 'keepalive', 'proxy', + 'proxy_password', 'proxy_username', + 'retries', 'throttle', 'timeout', 'mirrorlist', + 'cachedir' ] + vars.sort() + for attr in vars: + output = output + '%s = %s\n' % (attr, getattr(self, attr)) + output = output + 'baseurl =' + for url in self.urls: + output = output + ' %s\n' % url + + return output + + def check(self): + """self-check the repo information - if we don't have enough to move + on then raise a repo error""" + if len(self.urls) < 1: + raise Errors.RepoError, \ + 'Cannot find a valid deltaurl for repo: %s' % self.id + + def doProxyDict(self): + if self._proxy_dict: + return + + self._proxy_dict = {} # zap it + proxy_string = None + if self.proxy not in [None, '_none_']: + proxy_string = '%s' % self.proxy + if self.proxy_username is not None: + proxy_parsed = urlparse.urlsplit(self.proxy, allow_fragments=0) + proxy_proto = proxy_parsed[0] + proxy_host = proxy_parsed[1] + proxy_rest = proxy_parsed[2] + '?' + proxy_parsed[3] + proxy_string = '%s://%s@%s%s' % (proxy_proto, + self.proxy_username, proxy_host, proxy_rest) + + if self.proxy_password is not None: + proxy_string = '%s://%s:%s@%s%s' % (proxy_proto, + self.proxy_username, self.proxy_password, + proxy_host, proxy_rest) + + if proxy_string is not None: + self._proxy_dict['http'] = proxy_string + self._proxy_dict['https'] = proxy_string + self._proxy_dict['ftp'] = proxy_string + + def __headersListFromDict(self): + """Convert our dict of headers to a list of 2-tuples for urlgrabber.""" + headers = [] + + keys = self.http_headers.keys() + for key in keys: + headers.append((key, self.http_headers[key])) + + return headers + + def setupGrab(self): + """sets up the grabber functions with the already stocked in urls for + the mirror groups""" + + if self.failovermethod == 'roundrobin': + mgclass = urlgrabber.mirror.MGRandomOrder + else: + mgclass = urlgrabber.mirror.MirrorGroup + + headers = tuple(self.__headersListFromDict()) + + self.grabfunc = URLGrabber(keepalive=self.keepalive, + bandwidth=self.bandwidth, + retry=self.retries, + throttle=self.throttle, + progress_obj=self.callback, + proxies = self.proxy_dict, + failure_callback=self.failure_obj, + interrupt_callback=self.interrupt_callback, + timeout=self.timeout, + http_headers=headers, + reget='simple') + + self.grab = mgclass(self.grabfunc, self.urls, + failure_callback=self.mirror_failure_obj) + + def dirSetup(self): + """make the necessary dirs, if possible, raise on failure""" + + cachedir = os.path.join(self.parent.basecachedir, self.id) + deltasdir = os.path.join(cachedir, 'deltas') + self.parent.setAttribute('deltasdir', deltasdir) + self.setAttribute('cachedir', cachedir) + + cookie = cachedir + '/' + self.metadata_cookie_fn + self.setAttribute('metadata_cookie', cookie) + + for dir in [cachedir, self.parent.deltasdir]: + if self.cache == 0: + if os.path.exists(dir) and os.path.isdir(dir): + continue + else: + try: + os.makedirs(dir, mode=0755) + except OSError, e: + raise Errors.RepoError, \ + "Error making cache directory: %s error was: %s" % (dir, e) + else: + if not os.path.exists(dir): + raise Errors.RepoError, \ + "Cannot access repository dir %s" % dir + + def baseurlSetup(self): + """go through the baseurls and mirrorlists and populate self.urls + with valid ones, run self.check() at the end to make sure it worked""" + + goodurls = [] + if self.mirrorlist and not self.mirrorlistparsed: + mirrorurls = getMirrorList(self.mirrorlist, self.proxy_dict) + self.mirrorlistparsed = 1 + for url in mirrorurls: + url = parser.varReplace(url, self.yumvar) + self.baseurl.append(url) + + for url in self.baseurl: + url = parser.varReplace(url, self.yumvar) + (s,b,p,q,f,o) = urlparse.urlparse(url) + if s not in ['http', 'ftp', 'file', 'https']: + print 'not using ftp, http[s], or file for repos, skipping - %s' % (url) + continue + else: + goodurls.append(url) + + self.setAttribute('urls', goodurls) + self.check() + self.setupGrab() # update the grabber for the urls + + def __get(self, url=None, relative=None, local=None, start=None, end=None, + copy_local=0, checkfunc=None, text=None, reget='simple', cache=True): + """retrieve file from the mirrorgroup for the repo + relative to local, optionally get range from + start to end, also optionally retrieve from a specific baseurl""" + + # if local or relative is None: raise an exception b/c that shouldn't happen + # if url is not None - then do a grab from the complete url - not through + # the mirror, raise errors as need be + # if url is None do a grab via the mirror group/grab for the repo + # return the path to the local file + + # Turn our dict into a list of 2-tuples + headers = self.__headersListFromDict() + + # We will always prefer to send no-cache. + if not (cache or self.http_headers.has_key('Pragma')): + headers.append(('Pragma', 'no-cache')) + + headers = tuple(headers) + + if local is None or relative is None: + raise Errors.RepoError, \ + "get request for Repo %s, gave no source or dest" % self.id + + if self.cache == 1: + if os.path.exists(local): # FIXME - we should figure out a way + return local # to run the checkfunc from here + + else: # ain't there - raise + raise Errors.RepoError, \ + "Caching enabled but no local cache of %s from %s" % (local, + self) + if url is not None: + ug = URLGrabber(keepalive = self.keepalive, + bandwidth = self.bandwidth, + retry = self.retries, + throttle = self.throttle, + progress_obj = self.callback, + copy_local = copy_local, + reget = reget, + proxies = self.proxy_dict, + failure_callback = self.failure_obj, + interrupt_callback=self.interrupt_callback, + timeout=self.timeout, + checkfunc=checkfunc, + http_headers=headers, + ) + + remote = url + '/' + relative + + try: + result = ug.urlgrab(remote, local, + text=text, + range=(start, end), + ) + except URLGrabError, e: + raise Errors.RepoError, \ + "failed to retrieve %s from %s\nerror was %s" % (relative, self.id, e) + + else: + try: + result = self.grab.urlgrab(relative, local, + text = text, + range = (start, end), + copy_local=copy_local, + reget = reget, + checkfunc=checkfunc, + http_headers=headers, + ) + except URLGrabError, e: + raise Errors.RepoError, "failure: %s from %s: %s" % (relative, self.id, e) + + return result + + def getPackage(self, package, checkfunc = None, text = None, cache = True): + remote = package.deltarelativepath + local = package.deltalocalpath + if hasattr(package, 'basepath'): + basepath = package.basepath + else: + basepath = package.returnSimple('basepath') + + return self.__get(url=basepath, + relative=remote, + local=local, + checkfunc=checkfunc, + text=text, + cache=cache + ) + + def metadataCurrent(self): + """Check if there is a metadata_cookie and check its age. If the + age of the cookie is less than metadata_expire time then return true + else return False""" + + val = False + if os.path.exists(self.metadata_cookie): + cookie_info = os.stat(self.metadata_cookie) + if cookie_info[8] + self.metadata_expire > time.time(): + val = True + # WE ARE FROM THE FUTURE!!!! + elif cookie_info[8] > time.time(): + val = False + return val + + def setMetadataCookie(self): + """if possible, set touch the metadata_cookie file""" + + check = self.metadata_cookie + if not os.path.exists(self.metadata_cookie): + check = self.cachedir + + if os.access(check, os.W_OK): + fo = open(self.metadata_cookie, 'w+') + fo.close() + del fo + + + def setup(self, cache): + try: + self.cache = cache + self.baseurlSetup() + self.dirSetup() + except Errors.RepoError, e: + raise + + try: + self._loadRepoXML(text=self) + except Errors.RepoError, e: + raise Errors.RepoError, ('Cannot open/read %s file for repository: %s' % (self.repoMDFile, self)) + + + def _loadRepoXML(self, text=None): + """retrieve/check/read in repomd.xml from the repository""" + + remote = self.repoMDFile + if self.is_different: + local = self.cachedir + '/prestomd.xml' + else: + local = self.cachedir + '/repomd.xml' + + if self.repoXML is not None: + return + + if self.cache or self.metadataCurrent(): + if not os.path.exists(local): + raise Errors.RepoError, 'Cannot find %s file for %s' % (self.repoMDFile, self) + else: + result = local + else: + checkfunc = (self._checkRepoXML, (), {}) + try: + result = self.__get(relative=remote, + local=local, + copy_local=1, + text=text, + reget=None, + checkfunc=checkfunc, + cache=self.http_caching == 'all') + + + except URLGrabError, e: + raise Errors.RepoError, 'Error downloading file %s: %s' % (local, e) + # if we have a 'fresh' repomd.xml then update the cookie + self.setMetadataCookie() + + try: + self.repoXML = repoMDObject.RepoMD(self.id, result) + except Errors.RepoMDError, e: + raise Errors.RepoError, 'Error importing %s from %s: %s' % (self.repoMDFile, self, e) + + def _checkRepoXML(self, fo): + if type(fo) is types.InstanceType: + filepath = fo.filename + else: + filepath = fo + + try: + repoMDObject.RepoMD(self.id, filepath) + except Errors.RepoMDError, e: + raise URLGrabError(-1, 'Error importing %s for %s: %s' % (self.repoMDFile, self, e)) + + + def checkMD(self, fn, mdtype): + """check the metadata type against its checksum""" + + thisdata = self.repoXML.getData(mdtype) + + (r_ctype, r_csum) = thisdata.checksum # get the remote checksum + + if type(fn) == types.InstanceType: # this is an urlgrabber check + file = fn.filename + else: + file = fn + + try: + l_csum = self._checksum(r_ctype, file) # get the local checksum + except Errors.RepoError, e: + raise URLGrabError(-3, 'Error performing checksum') + + if l_csum == r_csum: + return 1 + else: + raise URLGrabError(-1, 'Metadata file does not match checksum') + + + + def retrieveMD(self, mdtype): + """base function to retrieve metadata files from the remote url + returns the path to the local metadata file of a 'mdtype' + mdtype must be 'deltas'.""" + try: + thisdata = self.repoXML.getData(mdtype) + except Errors.RepoMDError: + self.enabled = False + self.conduit.info(5, "No drpms available for %s" % self.id) + return + + (r_base, remote) = thisdata.location + fname = os.path.basename(remote) + local = self.cachedir + '/' + fname + + if self.retrieved.has_key(mdtype): + if self.retrieved[mdtype]: # got it, move along + return local + + if self.cache == 1: + if os.path.exists(local): + try: + self.checkMD(local, mdtype) + except URLGrabError, e: + raise Errors.RepoError, \ + "Caching enabled and local cache: %s does not match checksum" % local + else: + return local + + else: # ain't there - raise + raise Errors.RepoError, \ + "Caching enabled but no local cache of %s from %s" % (local, + self) + + if os.path.exists(local): + try: + self.checkMD(local, mdtype) + except URLGrabError, e: + pass + else: + self.retrieved[mdtype] = 1 + return local # it's the same return the local one + + try: + checkfunc = (self.checkMD, (mdtype,), {}) + local = self.__get(relative=remote, local=local, copy_local=1, + checkfunc=checkfunc, reget=None, + cache=self.http_caching == 'all') + except URLGrabError, e: + raise Errors.RepoError, \ + "Could not retrieve %s matching remote checksum from %s" % (local, self) + else: + self.retrieved[mdtype] = 1 + return local + + + def getPrestoXML(self): + """this gets you the path to the primary.xml file, retrieving it if we + need a new one""" + + return self.retrieveMD('deltas') + + def setCallback(self, callback): + self.callback = callback + self.setupGrab() + + def setFailureObj(self, failure_obj): + self.failure_obj = failure_obj + self.setupGrab() + + def setMirrorFailureObj(self, failure_obj): + self.mirror_failure_obj = failure_obj + self.setupGrab() + + def setInterruptCallback(self, callback): + self.interrupt_callback = callback + self.setupGrab() + +def getMirrorList(mirrorlist, pdict = None): + """retrieve an up2date-style mirrorlist file from a url, + we also s/$ARCH/$BASEARCH/ and move along + returns a list of the urls from that file""" + + returnlist = [] + if hasattr(urlgrabber.grabber, 'urlopen'): + urlresolver = urlgrabber.grabber + else: + import urllib + urlresolver = urllib + + scheme = urlparse.urlparse(mirrorlist)[0] + if scheme == '': + url = 'file://' + mirrorlist + else: + url = mirrorlist + + try: + fo = urlresolver.urlopen(url, proxies=pdict) + except urlgrabber.grabber.URLGrabError, e: + print "Could not retrieve mirrorlist %s error was\n%s" % (url, e) + fo = None + + if fo is not None: + content = fo.readlines() + for line in content: + if re.match('^\s*\#.*', line) or re.match('^\s*$', line): + continue + mirror = re.sub('\n$', '', line) # no more trailing \n's + (mirror, count) = re.subn('\$ARCH', '$BASEARCH', mirror) + returnlist.append(mirror) + + return returnlist + diff --git a/yum-presto-legacy/shared/prestoThread.py b/yum-presto-legacy/shared/prestoThread.py new file mode 100644 index 0000000..e090910 --- /dev/null +++ b/yum-presto-legacy/shared/prestoThread.py @@ -0,0 +1,53 @@ +# authors: Ahmed Kamal <email.ahmedkamal@googlemail.com> +# Jonathan Dieter <jdieter@gmail.com> +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2007 Ahmed Kamal, Jonathan Dieter + +import threading + +class ReconstructionThread(threading.Thread): + def __init__(self, queue, lock, run_function): + threading.Thread.__init__(self) + self.run_function = run_function + self.queue = queue + self.lock = lock + self.can_exit = False + self.messages = "" + + def run(self): + while True: + try: + retval = self.queue.get(not self.can_exit) + except: + # If we're done with our drpms and no more are coming, let's + # blow this joint + break + if retval != None: + messages = apply(self.run_function, retval) + if self.can_exit: + # If there are not going to be any more new drpms, + # send messages directly to conduit + conduit = retval[0] + if self.messages != "": + conduit.info(2, self.messages[:-1]) + self.messages = "" + if messages != "": + conduit.info(2, messages[:-1]) + else: + # We may be downloading drpms still, so queue messages + self.lock.acquire() + self.messages += messages + self.lock.release() diff --git a/yum-presto-legacy/shared/prestoTransaction.py b/yum-presto-legacy/shared/prestoTransaction.py new file mode 100644 index 0000000..ffaad01 --- /dev/null +++ b/yum-presto-legacy/shared/prestoTransaction.py @@ -0,0 +1,140 @@ +# author: Jonathan Dieter <jdieter@gmail.com> +# +# format_number taken almost completely from progress_meter.py in yum +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2005 Duke University +# Copyright 2007 Jonathan Dieter + +import os +import deltarpm + +def format_number(number, SI=False, space=''): + """Turn numbers into human-readable metric-like numbers""" + symbols = ['', # (none) + 'K', # kilo + 'M', # mega + 'G', # giga + 'T', # tera + 'P', # peta + 'E', # exa + 'Z', # zetta + 'Y'] # yotta + + if SI: step = 1000.0 + else: step = 1024.0 + + thresh = 999 + depth = 0 + + # we want numbers between + while number > thresh: + depth = depth + 1 + number = number / step + + # just in case someone needs more than 1000 yottabytes! + diff = depth - len(symbols) + 1 + if diff > 0: + depth = depth - diff + number = number * thresh**depth + + if type(number) == type(1) or type(number) == type(1L): + format = '%i%s%s' + elif number < 9.95: + # must use 9.95 for proper sizing. For example, 9.99 will be + # rounded to 10.0 with the .1f format string (which is too long) + format = '%.1f%s%s' + else: + format = '%.0f%s%s' + + return(format % (number, space, symbols[depth])) + + +def find_available_drpms(conduit, newpkg): + """Find any applicable drpms for newpkg + newpkg is a TransactionMember""" + + rpmdb = conduit.getRpmDB() + + is_local = False + + # Set p_repo to be packages delta repository or set to False if + # there is no delta repository + try: + p_repo = newpkg.po.repo.p_repo + drpm_enabled = p_repo.enabled + + po = newpkg.po + if hasattr(po, 'pkgtype') and po.pkgtype == 'local': + is_local = True + else: + local = po.localPkg() + if os.path.exists(local): + cursize = os.stat(local)[6] + totsize = long(po.size) + if not po.verifyLocalPkg(): + if cursize >= totsize: # otherwise keep it around for regetting + os.unlink(local) + else: + conduit.info(5, "using local copy of %s" % po) + is_local = True + + except: + conduit.info(5, "No Presto repository information for %s.%s %i:%s-%s" % (newpkg.name, newpkg.arch, int(newpkg.epoch), newpkg.version, newpkg.release)) + drpm_enabled = False + is_local = False + + chosen_drpm = None + + # First part of key when matching drpms + key1 = "%s*%s*%i*%s*%s" % (newpkg.name, newpkg.arch, int(newpkg.epoch), newpkg.version, newpkg.release) + + # Find any installed packages that match the ones we want to download + installed = rpmdb.searchNevra(newpkg.name, None, None, None, newpkg.arch) + + if installed == []: + is_installed = False + else: + is_installed = True + + + if is_installed and drpm_enabled and not is_local: + for oldpkg in installed: + # Generate second part of key for matching drpms, then full key + key2 = "%s*%s*%i*%s*%s" % (oldpkg.name, oldpkg.arch, int(oldpkg.epoch), oldpkg.version, oldpkg.release) + key = "%s!!%s" % (key1, key2) + + # Check whether we have a matching drpm + if p_repo.deltalist.has_key(key): + # Check whether or not we already have a matching drpm, then choose smallest of the two if we do + if chosen_drpm == None or p_repo.deltalist[key]['size'] < chosen_drpm['size']: + + # Get sequence code for drpm + sequence = p_repo.deltalist[key]['sequence'] + drpm = deltarpm.DeltaRpmWrapper(conduit) + + # Attempt to apply sequence code for drpm. If this fails, drpm will not apply cleanly, so + # don't even try to download it. + try: + drpm.verifySequence(newpkg.po.arch, sequence) + except: + conduit.info(5, "Verification of %s failed" % sequence) + else: + chosen_drpm = p_repo.deltalist[key] + chosen_drpm['baseurl'] = p_repo.baseurl[0] + newpkg.po.oldpkg_string = "%s.%s %s:%s-%s" % (oldpkg.name, oldpkg.arch, oldpkg.epoch, oldpkg.version, oldpkg.release) + newpkg.po.newpkg_string = "%s.%s %s:%s-%s" % (newpkg.name, newpkg.arch, newpkg.epoch, newpkg.version, newpkg.release) + + return (chosen_drpm, installed, is_local, drpm_enabled) diff --git a/yum-presto-legacy/shared/prestomdparser.py b/yum-presto-legacy/shared/prestomdparser.py new file mode 100644 index 0000000..6764f71 --- /dev/null +++ b/yum-presto-legacy/shared/prestomdparser.py @@ -0,0 +1,169 @@ +# author: Jonathan Dieter <jdieter@gmail.com> +# +# mostly taken from mdparser.py (part of yum) with a few minor modifications +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2005 Duke University +# Portions copyright 2007 Jonathan Dieter + +import gzip +try: + from cElementTree import iterparse +except: + from xml.etree.cElementTree import iterparse + +from cStringIO import StringIO + +#TODO: document everything here + +class PrestoMDParser: + + def __init__(self, filename): + + # Set up mapping of meta types to handler classes + handlers = { + '{http://linux.duke.edu/metadata/common}metadata': DeltasEntry, + } + + self.total = None + self.count = 0 + self._handlercls = None + + # Read in type, set package node handler and get total number of + # packages + if filename[-3:] == '.gz': fh = gzip.open(filename, 'r') + else: fh = open(filename, 'r') + parser = iterparse(fh, events=('start', 'end')) + self.reader = parser.__iter__() + event, elem = self.reader.next() + self._handlercls = handlers.get(elem.tag, None) + if not self._handlercls: + raise ValueError('Unknown repodata type "%s" in %s' % ( + elem.tag, filename)) + + def getDeltaList(self): + for event, elem in self.reader: + if event == 'end' and elem.tag == '{http://linux.duke.edu/metadata/common}metadata': + return self._handlercls(elem) + + +class BaseEntry: + def __init__(self, elem): + self._p = {} + + def __getitem__(self, k): + return self._p[k] + + def keys(self): + return self._p.keys() + + def values(self): + return self._p.values() + + def has_key(self, k): + return self._p.has_key(k) + + def __str__(self): + out = StringIO() + keys = self.keys() + keys.sort() + for k in keys: + line = u'%s=%s\n' % (k, self[k]) + out.write(line.encode('utf8')) + return out.getvalue() + + def _bn(self, qn): + if qn.find('}') == -1: return qn + return qn.split('}')[1] + + def _prefixprops(self, elem, prefix): + ret = {} + for key in elem.attrib.keys(): + ret[prefix + '_' + self._bn(key)] = elem.attrib[key] + return ret + +class DeltasEntry(BaseEntry): + def __init__(self, deltas): + BaseEntry.__init__(self, deltas) + # Avoid excess typing :) + p = self._p + + for elem in deltas: + temp = {} + key1 = "" + key2 = "" + for child in elem: + name = self._bn(child.tag) + if name in ('name', 'arch'): + temp[name] = child.text + + elif name == 'version': + attrib = child.attrib + try: + attrib['epoch'] = int(attrib['epoch']) + except: + attrib['epoch'] = 0 + key1 = "%s*%s*%i*%s*%s" % (temp['name'], temp['arch'], attrib['epoch'], attrib['ver'], attrib['rel']) + + elif name == 'deltas': + for oldrpm in child: + temp2 = {} + value = {} + key = None + for oldrpm_child in oldrpm: + name = self._bn(oldrpm_child.tag) + if name in ('name', 'arch'): + temp2[name] = oldrpm_child.text + + elif name == 'version': + ch_attrib = oldrpm_child.attrib + try: + ch_attrib['epoch'] = int(ch_attrib['epoch']) + except: + ch_attrib['epoch'] = attrib['epoch'] + try: + ch_attrib['ver'] = ch_attrib['ver'] + except: + ch_attrib['ver'] = attrib['ver'] + if not temp2.has_key('name'): + temp2['name'] = temp['name'] + if not temp2.has_key('arch'): + temp2['arch'] = temp['arch'] + key2 = "%s*%s*%i*%s*%s" % (temp2['name'], temp2['arch'], ch_attrib['epoch'], ch_attrib['ver'], ch_attrib['rel']) + key = "%s!!%s" % (key1, key2) + p[key] = {} + + if name in ('sequence', 'drpm_filename', 'size'): + p[key][name] = oldrpm_child.text + + if name == "checksum": + p[key][name] = oldrpm_child.text + p[key]["%s_type" % name] = oldrpm_child.attrib['type'] + deltas.clear() + +def test(): + import sys + + parser = PrestoMDParser(sys.argv[1]) + + deltalist = parser.getDeltaList() + + print '-' * 40 + print deltalist + + print 'read: %s deltarpms ' % (len(deltalist.keys())) + +if __name__ == '__main__': + test() |