From 5ea857b0e948b687785b8e55e08866c6171fb715 Mon Sep 17 00:00:00 2001 From: Jonathan Dieter Date: Thu, 29 Mar 2007 19:43:26 +0300 Subject: Split server and client Signed-off-by: Jonathan Dieter --- shared/prestoRepo.py | 612 --------------------------------------------------- 1 file changed, 612 deletions(-) delete mode 100644 shared/prestoRepo.py (limited to 'shared/prestoRepo.py') diff --git a/shared/prestoRepo.py b/shared/prestoRepo.py deleted file mode 100644 index 582dc2f..0000000 --- a/shared/prestoRepo.py +++ /dev/null @@ -1,612 +0,0 @@ -# author: Jonathan Dieter -# -# mostly taken from yumRepo.py (part of yum) with a few minor modifications -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 2 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Library General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. -# Copyright 2005 Duke University - -import os -import re -import time -import types -import urlparse - -from yum import Errors -from urlgrabber.grabber import URLGrabber -import urlgrabber.mirror -from urlgrabber.grabber import URLGrabError -from yum.repos import Repository -from yum import repoMDObject -from yum import parser -from yum import config -from yum import misc - -class PrestoRepository(Repository): - """ - This is an actual repository object - - Configuration attributes are pulled in from config.RepoConf. - """ - - def __init__(self, repo, conduit): - Repository.__init__(self, repo.id) - - # If there's a specific deltarpm url, use that - is_different = False - if conduit.confString(repo.id, 'deltaurl'): - self.baseurl = [conduit.confString(repo.id, 'deltaurl')] - is_different = True - conduit.info(5, 'Manual url set from presto.conf: %s' % self.baseurl) - elif repo.deltaurl != []: - self.baseurl = repo.deltaurl - is_different = True - conduit.info(5, 'Manual url set from repository conf file: %s' % self.baseurl) - else: - self.baseurl = repo.baseurl - - # If there's a specific mirrorlist, use that - if conduit.confString(repo.id, 'deltamirrorlist'): - self.mirrorlist = conduit.confString(repo.id, 'deltamirrorlist') - self.baseurl = None - is_different = True - conduit.info(5, 'Manual mirrorlist set from presto.conf: %s' % self.mirrorlist) - elif repo.deltamirrorlist != None: - self.mirrorlist = repo.deltamirrorlist - self.baseurl = None - is_different = True - conduit.info(5, 'Manual mirrorlist set from repository conf file: %s' % self.mirrorlist) - else: - if self.baseurl == repo.baseurl: - self.mirrorlist = repo.mirrorlist - else: - self.mirrorlist = None - - self.conduit = conduit - self.urls = [] - self.is_different = is_different - if is_different: - self.repoMDFile = 'repodata/prestomd.xml' - self.metadata_cookie_fn = 'presto_cachecookie' - else: - self.repoMDFile = 'repodata/repomd.xml' - self.metadata_cookie_fn = 'cachecookie' - self.repoXML = None - self.cache = 0 - self.mirrorlistparsed = 0 - self.yumvar = {} # empty dict of yumvariables for $string replacement - self._proxy_dict = {} - self.http_headers = {} - - # throw in some stubs for things that will be set by the config class - self.basecachedir = "" - self.cachedir = "" - self.pkgdir = "" - self.hdrdir = "" - self.enabled = True - - # holder for stuff we've grabbed - self.retrieved = { 'deltas':0 } - - # callbacks - self.keepalive = repo.keepalive - self.bandwidth = repo.bandwidth - self.retries = repo.retries - self.throttle = repo.throttle - self.proxy = repo.proxy - self.proxy_username = repo.proxy_username - self.proxy_password = repo.proxy_password - self.timeout = repo.timeout - self.http_caching = repo.http_caching - self.failovermethod = repo.failovermethod - self.metadata_expire = repo.metadata_expire - self.basecachedir = repo.basecachedir - self.callback = repo.callback - self.failure_obj = repo.failure_obj - self.mirror_failure_obj = repo.mirror_failure_obj - self.interrupt_callback = repo.interrupt_callback - self.drpm_list = {} - self.parent = repo - repo.p_repo = self - - - def __getProxyDict(self): - self.doProxyDict() - if self._proxy_dict: - return self._proxy_dict - return None - - # consistent access to how proxy information should look (and ensuring - # that it's actually determined for the repo) - proxy_dict = property(__getProxyDict) - - def ready(self): - """Returns true if this repository is setup and ready for use.""" - return self.repoXML is not None - - def __cmp__(self, other): - if self.id > other.id: - return 1 - elif self.id < other.id: - return -1 - else: - return 0 - - def __str__(self): - return self.id - - def _checksum(self, sumtype, file, CHUNK=2**16): - """takes filename, hand back Checksum of it - sumtype = md5 or sha - filename = /path/to/file - CHUNK=65536 by default""" - try: - return misc.checksum(sumtype, file, CHUNK) - except (Errors.MiscError, EnvironmentError), e: - raise Errors.RepoError, 'Error opening file for checksum: %s' % e - - def dump(self): - output = '[%s]\n' % self.id - vars = ['id', 'bandwidth', 'enabled', - 'keepalive', 'proxy', - 'proxy_password', 'proxy_username', - 'retries', 'throttle', 'timeout', 'mirrorlist', - 'cachedir' ] - vars.sort() - for attr in vars: - output = output + '%s = %s\n' % (attr, getattr(self, attr)) - output = output + 'baseurl =' - for url in self.urls: - output = output + ' %s\n' % url - - return output - - def check(self): - """self-check the repo information - if we don't have enough to move - on then raise a repo error""" - if len(self.urls) < 1: - raise Errors.RepoError, \ - 'Cannot find a valid deltaurl for repo: %s' % self.id - - def doProxyDict(self): - if self._proxy_dict: - return - - self._proxy_dict = {} # zap it - proxy_string = None - if self.proxy not in [None, '_none_']: - proxy_string = '%s' % self.proxy - if self.proxy_username is not None: - proxy_parsed = urlparse.urlsplit(self.proxy, allow_fragments=0) - proxy_proto = proxy_parsed[0] - proxy_host = proxy_parsed[1] - proxy_rest = proxy_parsed[2] + '?' + proxy_parsed[3] - proxy_string = '%s://%s@%s%s' % (proxy_proto, - self.proxy_username, proxy_host, proxy_rest) - - if self.proxy_password is not None: - proxy_string = '%s://%s:%s@%s%s' % (proxy_proto, - self.proxy_username, self.proxy_password, - proxy_host, proxy_rest) - - if proxy_string is not None: - self._proxy_dict['http'] = proxy_string - self._proxy_dict['https'] = proxy_string - self._proxy_dict['ftp'] = proxy_string - - def __headersListFromDict(self): - """Convert our dict of headers to a list of 2-tuples for urlgrabber.""" - headers = [] - - keys = self.http_headers.keys() - for key in keys: - headers.append((key, self.http_headers[key])) - - return headers - - def setupGrab(self): - """sets up the grabber functions with the already stocked in urls for - the mirror groups""" - - if self.failovermethod == 'roundrobin': - mgclass = urlgrabber.mirror.MGRandomOrder - else: - mgclass = urlgrabber.mirror.MirrorGroup - - headers = tuple(self.__headersListFromDict()) - - self.grabfunc = URLGrabber(keepalive=self.keepalive, - bandwidth=self.bandwidth, - retry=self.retries, - throttle=self.throttle, - progress_obj=self.callback, - proxies = self.proxy_dict, - failure_callback=self.failure_obj, - interrupt_callback=self.interrupt_callback, - timeout=self.timeout, - http_headers=headers, - reget='simple') - - self.grab = mgclass(self.grabfunc, self.urls, - failure_callback=self.mirror_failure_obj) - - def dirSetup(self): - """make the necessary dirs, if possible, raise on failure""" - - cachedir = os.path.join(self.parent.basecachedir, self.id) - deltasdir = os.path.join(cachedir, 'deltas') - self.parent.setAttribute('deltasdir', deltasdir) - self.setAttribute('cachedir', cachedir) - - cookie = cachedir + '/' + self.metadata_cookie_fn - self.setAttribute('metadata_cookie', cookie) - - for dir in [cachedir, self.parent.deltasdir]: - if self.cache == 0: - if os.path.exists(dir) and os.path.isdir(dir): - continue - else: - try: - os.makedirs(dir, mode=0755) - except OSError, e: - raise Errors.RepoError, \ - "Error making cache directory: %s error was: %s" % (dir, e) - else: - if not os.path.exists(dir): - raise Errors.RepoError, \ - "Cannot access repository dir %s" % dir - - def baseurlSetup(self): - """go through the baseurls and mirrorlists and populate self.urls - with valid ones, run self.check() at the end to make sure it worked""" - - goodurls = [] - if self.mirrorlist and not self.mirrorlistparsed: - mirrorurls = getMirrorList(self.mirrorlist, self.proxy_dict) - self.mirrorlistparsed = 1 - for url in mirrorurls: - url = parser.varReplace(url, self.yumvar) - self.baseurl.append(url) - - for url in self.baseurl: - url = parser.varReplace(url, self.yumvar) - (s,b,p,q,f,o) = urlparse.urlparse(url) - if s not in ['http', 'ftp', 'file', 'https']: - print 'not using ftp, http[s], or file for repos, skipping - %s' % (url) - continue - else: - goodurls.append(url) - - self.setAttribute('urls', goodurls) - self.check() - self.setupGrab() # update the grabber for the urls - - def __get(self, url=None, relative=None, local=None, start=None, end=None, - copy_local=0, checkfunc=None, text=None, reget='simple', cache=True): - """retrieve file from the mirrorgroup for the repo - relative to local, optionally get range from - start to end, also optionally retrieve from a specific baseurl""" - - # if local or relative is None: raise an exception b/c that shouldn't happen - # if url is not None - then do a grab from the complete url - not through - # the mirror, raise errors as need be - # if url is None do a grab via the mirror group/grab for the repo - # return the path to the local file - - # Turn our dict into a list of 2-tuples - headers = self.__headersListFromDict() - - # We will always prefer to send no-cache. - if not (cache or self.http_headers.has_key('Pragma')): - headers.append(('Pragma', 'no-cache')) - - headers = tuple(headers) - - if local is None or relative is None: - raise Errors.RepoError, \ - "get request for Repo %s, gave no source or dest" % self.id - - if self.cache == 1: - if os.path.exists(local): # FIXME - we should figure out a way - return local # to run the checkfunc from here - - else: # ain't there - raise - raise Errors.RepoError, \ - "Caching enabled but no local cache of %s from %s" % (local, - self) - if url is not None: - ug = URLGrabber(keepalive = self.keepalive, - bandwidth = self.bandwidth, - retry = self.retries, - throttle = self.throttle, - progress_obj = self.callback, - copy_local = copy_local, - reget = reget, - proxies = self.proxy_dict, - failure_callback = self.failure_obj, - interrupt_callback=self.interrupt_callback, - timeout=self.timeout, - checkfunc=checkfunc, - http_headers=headers, - ) - - remote = url + '/' + relative - - try: - result = ug.urlgrab(remote, local, - text=text, - range=(start, end), - ) - except URLGrabError, e: - raise Errors.RepoError, \ - "failed to retrieve %s from %s\nerror was %s" % (relative, self.id, e) - - else: - try: - result = self.grab.urlgrab(relative, local, - text = text, - range = (start, end), - copy_local=copy_local, - reget = reget, - checkfunc=checkfunc, - http_headers=headers, - ) - except URLGrabError, e: - raise Errors.RepoError, "failure: %s from %s: %s" % (relative, self.id, e) - - return result - - def getPackage(self, package, checkfunc = None, text = None, cache = True): - remote = package.returnSimple('relativepath') - local = package.localPkg() - basepath = package.returnSimple('basepath') - - return self.__get(url=basepath, - relative=remote, - local=local, - checkfunc=checkfunc, - text=text, - cache=cache - ) - - def metadataCurrent(self): - """Check if there is a metadata_cookie and check its age. If the - age of the cookie is less than metadata_expire time then return true - else return False""" - - val = False - if os.path.exists(self.metadata_cookie): - cookie_info = os.stat(self.metadata_cookie) - if cookie_info[8] + self.metadata_expire > time.time(): - val = True - # WE ARE FROM THE FUTURE!!!! - elif cookie_info[8] > time.time(): - val = False - return val - - def setMetadataCookie(self): - """if possible, set touch the metadata_cookie file""" - - check = self.metadata_cookie - if not os.path.exists(self.metadata_cookie): - check = self.cachedir - - if os.access(check, os.W_OK): - fo = open(self.metadata_cookie, 'w+') - fo.close() - del fo - - - def setup(self, cache): - try: - self.cache = cache - self.baseurlSetup() - self.dirSetup() - except Errors.RepoError, e: - raise - - try: - self._loadRepoXML(text=self) - except Errors.RepoError, e: - raise Errors.RepoError, ('Cannot open/read %s file for repository: %s' % (self.repoMDFile, self)) - - - def _loadRepoXML(self, text=None): - """retrieve/check/read in repomd.xml from the repository""" - - remote = self.repoMDFile - if self.is_different: - local = self.cachedir + '/prestomd.xml' - else: - local = self.cachedir + '/repomd.xml' - - if self.repoXML is not None: - return - - if self.cache or self.metadataCurrent(): - if not os.path.exists(local): - raise Errors.RepoError, 'Cannot find %s file for %s' % (self.repoMDFile, self) - else: - result = local - else: - checkfunc = (self._checkRepoXML, (), {}) - try: - result = self.__get(relative=remote, - local=local, - copy_local=1, - text=text, - reget=None, - checkfunc=checkfunc, - cache=self.http_caching == 'all') - - - except URLGrabError, e: - raise Errors.RepoError, 'Error downloading file %s: %s' % (local, e) - # if we have a 'fresh' repomd.xml then update the cookie - self.setMetadataCookie() - - try: - self.repoXML = repoMDObject.RepoMD(self.id, result) - except Errors.RepoMDError, e: - raise Errors.RepoError, 'Error importing %s from %s: %s' % (self.repoMDFile, self, e) - - def _checkRepoXML(self, fo): - if type(fo) is types.InstanceType: - filepath = fo.filename - else: - filepath = fo - - try: - repoMDObject.RepoMD(self.id, filepath) - except Errors.RepoMDError, e: - raise URLGrabError(-1, 'Error importing %s for %s: %s' % (self.repoMDFile, self, e)) - - - def checkMD(self, fn, mdtype): - """check the metadata type against its checksum""" - - thisdata = self.repoXML.getData(mdtype) - - (r_ctype, r_csum) = thisdata.checksum # get the remote checksum - - if type(fn) == types.InstanceType: # this is an urlgrabber check - file = fn.filename - else: - file = fn - - try: - l_csum = self._checksum(r_ctype, file) # get the local checksum - except Errors.RepoError, e: - raise URLGrabError(-3, 'Error performing checksum') - - if l_csum == r_csum: - return 1 - else: - raise URLGrabError(-1, 'Metadata file does not match checksum') - - - - def retrieveMD(self, mdtype): - """base function to retrieve metadata files from the remote url - returns the path to the local metadata file of a 'mdtype' - mdtype must be 'deltas'.""" - try: - thisdata = self.repoXML.getData(mdtype) - except Errors.RepoMDError: - self.enabled = False - self.conduit.info(5, "No drpms available for %s" % self.id) - return - - (r_base, remote) = thisdata.location - fname = os.path.basename(remote) - local = self.cachedir + '/' + fname - - if self.retrieved.has_key(mdtype): - if self.retrieved[mdtype]: # got it, move along - return local - - if self.cache == 1: - if os.path.exists(local): - try: - self.checkMD(local, mdtype) - except URLGrabError, e: - raise Errors.RepoError, \ - "Caching enabled and local cache: %s does not match checksum" % local - else: - return local - - else: # ain't there - raise - raise Errors.RepoError, \ - "Caching enabled but no local cache of %s from %s" % (local, - self) - - if os.path.exists(local): - try: - self.checkMD(local, mdtype) - except URLGrabError, e: - pass - else: - self.retrieved[mdtype] = 1 - return local # it's the same return the local one - - try: - checkfunc = (self.checkMD, (mdtype,), {}) - local = self.__get(relative=remote, local=local, copy_local=1, - checkfunc=checkfunc, reget=None, - cache=self.http_caching == 'all') - except URLGrabError, e: - raise Errors.RepoError, \ - "Could not retrieve %s matching remote checksum from %s" % (local, self) - else: - self.retrieved[mdtype] = 1 - return local - - - def getPrestoXML(self): - """this gets you the path to the primary.xml file, retrieving it if we - need a new one""" - - return self.retrieveMD('deltas') - - def setCallback(self, callback): - self.callback = callback - self.setupGrab() - - def setFailureObj(self, failure_obj): - self.failure_obj = failure_obj - self.setupGrab() - - def setMirrorFailureObj(self, failure_obj): - self.mirror_failure_obj = failure_obj - self.setupGrab() - - def setInterruptCallback(self, callback): - self.interrupt_callback = callback - self.setupGrab() - -def getMirrorList(mirrorlist, pdict = None): - """retrieve an up2date-style mirrorlist file from a url, - we also s/$ARCH/$BASEARCH/ and move along - returns a list of the urls from that file""" - - returnlist = [] - if hasattr(urlgrabber.grabber, 'urlopen'): - urlresolver = urlgrabber.grabber - else: - import urllib - urlresolver = urllib - - scheme = urlparse.urlparse(mirrorlist)[0] - if scheme == '': - url = 'file://' + mirrorlist - else: - url = mirrorlist - - try: - fo = urlresolver.urlopen(url, proxies=pdict) - except urlgrabber.grabber.URLGrabError, e: - print "Could not retrieve mirrorlist %s error was\n%s" % (url, e) - fo = None - - if fo is not None: - content = fo.readlines() - for line in content: - if re.match('^\s*\#.*', line) or re.match('^\s*$', line): - continue - mirror = re.sub('\n$', '', line) # no more trailing \n's - (mirror, count) = re.subn('\$ARCH', '$BASEARCH', mirror) - returnlist.append(mirror) - - return returnlist - -- cgit