diff options
Diffstat (limited to 'deltarpm.py')
-rw-r--r-- | deltarpm.py | 513 |
1 files changed, 0 insertions, 513 deletions
diff --git a/deltarpm.py b/deltarpm.py deleted file mode 100644 index aac0c02..0000000 --- a/deltarpm.py +++ /dev/null @@ -1,513 +0,0 @@ -# author: Lars Herrmann <herrmann@redhat.com> -# Ahmed Kamal <email.ahmedkamal@googlemail.com> -# license: GPL (see COPYING file in distribution) -# -# this module provides a python wrapper around deltarpm tools written by suse -# -# TODO: catch exceptions wherever possible and raise useful ones ;) -# see TODO lines in methods - -MAKE='/usr/bin/makedeltarpm' -APPLY='/usr/bin/applydeltarpm' -SEQ_SUFFIX='seq' -SUFFIX='drpm' - -# constants for up2date configuration -# major flag to enable or disable use of delta rpms , used in patched up2date -USE_DELTA='useDeltaRpms' -# directory where to store generated rpms - depending on use up2date storage or temp storage for fetching -STORAGE='storageDir' -# directory where to store delta rpm files, normally temporarily -DELTA_STORAGE='deltaStorageDir' -# url to fetch delta rpm files from -DELTA_URL='deltaRpmURL' -# directory where local copies of oldrpms reside -DELTA_OLDRPM_REPOSITORY='deltaOldRpmRepository' -# flag indicates if downloading and verfying sequence files before downloading delta rpms - saves bandwidth -DELTA_USE_SEQ='deltaUseSequences' -# flag indicates if local copies of old rpms should be used -DELTA_USE_OLDRPMS='deltaUseOldRpms' -SIZELIMIT='deltaRpmSizeLimit' - -# default setting for STORAGE - same as with up2date -DEFAULT_STORAGE='/var/spool/up2date' -# default setting for DELTA_STORAGE -DEFAULT_DELTA_STORAGE='/var/spool/up2date/deltarpms' - -# enable or disable to see verbose messages on stdout -DEBUG=0 - -import popen2 -import string -import os -import glob - -class Process: - """wrapper class to execute programs and return exitcode and output (stdout and stderr combined)""" - def __init__(self): - self.__stdout=None - self.__returncode=None - self.__command=None - self.__args=None - - def run(self, command, *args): - self.__command=command - self.__args=args - cmdline=command+" "+string.join(args, " ") - if DEBUG: - print 'DEBUG: %s.%s: executing %s' % (self.__class__, 'run', cmdline) - pipe = popen2.Popen4(cmdline) - self.__stdout=pipe.fromchild.read() - retcode = pipe.wait() - if os.WIFEXITED(retcode): - self.__returncode = os.WEXITSTATUS(retcode) - else: - self.__returncode = retcode - # fallback to old implementation - works better ? - #stdoutp = os.popen(cmdline,'r',1) - #self.__stdout = stdoutp.read() - #retcode = stdoutp.close() - #if retcode is None: - # self.__returncode = 0 - #else: - # self.__returncode = retcode - - def getOutput(self): - return self.__stdout - - def returnCode(self): - return self.__returncode - -class RpmDescription: - """Wrapper class to encapsulate RPM attributes""" - - def __init__(self, name,version,release,arch, epoch=''): - """constructor: provide major attributes in correct order - epoch optional""" - self.name=name - self.version=version - self.release=release - self.epoch=epoch - self.arch=arch - if DEBUG: - print 'DEBUG: %s.%s: created: %s' % (self.__class__, '__init__', self) - - def rhnFileName(self): - """return file name in e:nvr.a notation""" - if self.epoch: - return "%s:%s-%s-%s.%s" % (self.epoch, self.name, self.version, self.release, self.arch) - else: - return "%s-%s-%s.%s" % (self.name, self.version, self.release, self.arch) - - def evr(self): - """return file name in e:vr notation as used in Satellite repository""" - if self.epoch: - return "%s:%s-%s" % (self.epoch, self.version, self.release) - else: - return "%s-%s" % (self.version, self.release) - - def satellitePath(self): - """return file path as used in Satellite repository - relative to /var/satellite""" - return "%s/%s/%s/%s.rpm" % (self.name, self.evr(), self.arch, self.fileName()) - - def fileName(self): - """return file name in nvr.a notation as used in up2date storageDir""" - return "%s-%s-%s.%s" % (self.name, self.version, self.release, self.arch) - - def __str__(self): - return self.rhnFileName() - -class DeltaRpmWrapper: - """wrapper around deltarpm binaries - implement methods for create, apply and verify delta rpms - - raises exceptions if exitcode of binaries was != 0""" - - def __init__(self, storageDir, oldRpmDir=None): - """constructor - params: storageDir=path of delta rpm storage, oldRpmDir = path of full rpm repository (optional)""" - self.storageDir = storageDir - self.oldRpmDir = oldRpmDir - if DEBUG: - print 'DEBUG: %s.%s: created: %s' % (self.__class__, '__init__', self) - - def __str__(self): - return "%s: storageDir=%s, oldRpmDir=%s" % (self.__class__, self.storageDir, self.oldRpmDir) - - def create(self, oldrpm, newrpm): - """wraps execution of makedeltarpm -s seqfile oldrpm newrpm deltarpm - constructs file names and paths based on given RpmDescription and instance settings for directories""" - if DEBUG: - print 'DEBUG: %s.create(%s,%s)' % (self.__class__,oldrpm, newrpm) - - # contruct filenames in satellite repository - oldrpmfile = "%s/%s" % (self.oldRpmDir, oldrpm.satellitePath()) - newrpmfile = "%s/%s" % (self.oldRpmDir, newrpm.satellitePath()) - - # check if file exists - # this is an ugly workaround, where the epoch is 0, but satellite - # stores it as 0:package, so we try it with epoch = string "0" - # we do glob.glob here, because satellite path could also contain - # shell wildcards - if not glob.glob(oldrpmfile): - if not oldrpm.epoch: - oldrpm.epoch = '0' - oldrpmfile = "%s/%s" % (self.oldRpmDir, oldrpm.satellitePath()) - if not glob.glob(newrpmfile): - if not newrpm.epoch: - newrpm.epoch = '0' - newrpmfile = "%s/%s" % (self.oldRpmDir, newrpm.satellitePath()) - - # construct filenames in deltarpm repository: - # /root/oldrpm/newrpm.{seq|rpm} - # with oldrpm|newrpm in e:nvr.a notation - deltarpm = newrpm.rhnFileName() - # files should go to /root/oldrpm - deltadir = "%s/%s" % (self.storageDir, oldrpm.rhnFileName()) - # TODO check if is a directory - if not os.access(deltadir, os.F_OK): - if DEBUG: - print 'DEBUG: %s.create: mkdir(%s)' % (__name__, deltadir) - os.makedirs(deltadir) - # filenames - deltarpmfile = "%s/%s" % (deltadir, deltarpm) - p=Process() - p.run(MAKE, '-s', "%s.%s" % (deltarpmfile,SEQ_SUFFIX), oldrpmfile, newrpmfile, "%s.rpm" % deltarpmfile) - # save output into logfile - logfile = "%s.log" % deltarpmfile - fd = open(logfile,'w') - fd.write(p.getOutput()) - print >> fd, "of: %s \nnf: %s" % (oldrpmfile, newrpmfile) - - fd.close() - if p.returnCode(): - raise Exception("%s.create: exitcode was %s - see %s" % (self.__class__,p.returnCode(), logfile)) - return deltarpmfile - - def apply(self, oldrpm, newrpm, deltarpmfile, useOldRpms = 0): - """wraps execution of applydeltarpm [-r oldrpm] deltarpm newrpm - - constructs file names and paths based on given RpmDescription and instance settings for directories""" - # args: RpmDescription - # TODO: test args for type == instance and __class__ == RpmDescription - # TODO: test without useOldRpms - if DEBUG: - print 'DEBUG: %s.apply(%s,%s,%s,%s)' % (self.__class__,oldrpm, newrpm, deltarpmfile, useOldRpms) - p=Process() - # targetrpm filename - newrpmfile = "%s/%s-%s-%s.%s.rpm" % (self.storageDir, newrpm.name, newrpm.version, newrpm.release, newrpm.arch) - if useOldRpms: - # TODO: check if self.oldRpmDir is set and exists ! - oldrpmfile = "%s/%s-%s-%s.%s.rpm" % (self.oldRpmDir, oldrpm.name, oldrpm.version, oldrpm.release, oldrpm.arch) - p.run(APPLY, '-r', oldrpmfile, deltarpmfile, newrpmfile) - else: - p.run(APPLY, deltarpmfile, newrpmfile) - if p.returnCode(): - # in case of error save output into logfile - will not be removed for further inspection - logfile = "%s.log" % deltarpmfile - fd = open(logfile,'w') - fd.write(p.getOutput()) - fd.close() - raise Exception("%s.apply(%s) exitcode was %d - see %s" % (self.__class__, newrpm, p.returnCode(), logfile)) - return newrpmfile - - def verifySequence(self, sequencefile, oldrpm = None, useOldRpms = 0): - """wraps execution of applydeltarpm [-r oldrpm] -s seqfilecontent - - constructs file names and paths based on given RpmDescription and instance settings for directories""" - if DEBUG: - print 'DEBUG: %s.verify(%s,%s,%s)' % (self.__class__,sequencefile, oldrpm, useOldRpms) - # read sequencefile - fd = open(sequencefile) - # TODO: is strip safe here ? could remove other chars than the linebreak - content = string.strip(string.join(fd.readlines())) - fd.close() - p = Process() - if useOldRpms: - oldrpmfile = "%s/%s-%s-%s.%s.rpm" % (self.oldRpmDir, oldrpm.name, oldrpm.version, oldrpm.release, oldrpm.arch) - p.run(APPLY, '-s', content, '-r', oldrpmfile) - else: - p.run(APPLY, '-s', content) - if p.returnCode(): - # in case of error save output into logfile - will not be removed for further inspection - logfile = "%s.log" % deltarpmfile - fd = open(logfile,'w') - fd.write(p.getOutput()) - fd.close() - raise Exception("could not verify sequence of delta rpm: %d - see %s" % (p.returnCode(), logfile)) -class Fetcher: - """ abstract class to be derived from classes implementing fetching seq and rpm files """ - - def fetchSequence(self, oldrpm, targetrpm): - pass - - def fetchDeltaRpm(self, oldrpm, targetrpm): - pass - -class HttpFetcher(Fetcher): - """ fetching seq and rpm files via http urls""" - - def __init__(self, deltaUrl, destinationDir): - """constructor - params: deltaUrl = webapp-url, destinationDir=path to store files""" - self.deltaUrl = deltaUrl - self.destinationDir = destinationDir - if DEBUG: - print 'DEBUG: %s.%s: created: %s' % (self.__class__, '__init__', self) - - def fetchSequence(self, oldrpm, targetrpm): - if DEBUG: - print 'DEBUG: %s.fetchSequence: : (%s,%s)' % (self.__class__, oldrpm, targetrpm) - return self.__fetchFile(oldrpm, targetrpm, SEQ_SUFFIX, 1) - - # The following method has been disabled by Fedora Infrastructure team, as - # we will not be using a server side web service, rather, delta rpms will be generated - # periodically, and client side, will simply download them if applicable - def __DISABLED__fetchFile(self, oldrpm, targetrpm, suffix, sequence=0): - """private method - uses private module to do the http request to rely on http return code""" - import httppost - data={} - data['oldname'] = oldrpm.name - data['oldversion'] = oldrpm.version - data['oldrelease'] = oldrpm.release - # avoid that httplib would send 'None' and not empty string - if oldrpm.epoch: - data['oldepoch'] = oldrpm.epoch - else: - data['oldepoch']='' - data['oldarch'] = oldrpm.arch - data['newname'] = targetrpm.name - data['newversion'] = targetrpm.version - data['newrelease'] = targetrpm.release - if targetrpm.epoch: - data['newepoch'] = targetrpm.epoch - else: - data['newepoch'] = '' - data['newarch'] = targetrpm.arch - if sequence: - data['sequence']='1' - - fd = httppost.send(self.deltaUrl, data, DEBUG) - content = fd.read() - fd.close() - dest = "%s/%s-%s-%s.%s.%s" % (self.destinationDir, targetrpm.name, targetrpm.version, targetrpm.release, targetrpm.arch, suffix) - fd=open(dest,'w') - fd.write(content) - fd.close - return dest - def __fetchFile(self, oldrpm, targetrpm, suffix, sequence=0): - """private method - uses private module to download delta rpms""" - import urllib2 - - if sequence: - data['sequence']='1' - - drpmName = getDrpmName(oldrpm, targetrpm) - fullUrl = '%s%s.%s' % (self.deltaUrl,drpmName,suffix) - if DEBUG: - print 'DEBUG: oldrpm: %s, newrpm: %s, suffix: %s' % (oldrpm, targetrpm, suffix) - print 'DEBUG: %s.__fetchFile: : (%s)' % (self.__class__, fullUrl) - try: - fd = urllib2.urlopen(fullUrl) - except IOError, e: - if hasattr(e, 'reason'): - raise Exception ("Failed to download delta rpm from URL %s, error: %s" % (fullUrl,e.reason)) - elif hasattr(e, 'code'): - raise Exception ("Failed to download delta rpm from URL %s, error: %s" % (fullUrl,e.code)) - else: - content = fd.read() - fd.close() - dest = "%s/%s-%s-%s.%s.%s" % (self.destinationDir, targetrpm.name, targetrpm.version, targetrpm.release, targetrpm.arch, suffix) - fd=open(dest,'w') - fd.write(content) - fd.close - return dest - - def fetchDeltaRpm(self, oldrpm, targetrpm): - if DEBUG: - print 'DEBUG: %s.fetchDeltaRpm: : (%s,%s)' % (self.__class__, oldrpm, targetrpm) - return self.__fetchFile(oldrpm, targetrpm, SUFFIX, 0) - - def __str__(self): - return "%s: deltaUrl=%s, destinationDir=%s" % (self.__class__, self.deltaUrl, self.destinationDir) - -class TestFSFetcher(Fetcher): - """ fetching seq and rpm files from local filesystem - uses NOT same directory structure as DeltaRpmWrapper.create""" - - def __init__(self, sourceDir, destinationDir): - self.sourceDir = sourceDir - self.destinationDir = destinationDir - if DEBUG: - print 'DEBUG: %s.%s: created: %s' % (self.__class__, '__init__', self) - - def __str__(self): - return "%s: sourceDir=%s, destinationDir=%s" % (self.__class__, self.sourceDir, self.destinationDir) - - def __copyFile(self, targetrpm, suffix): - source = "%s/%s-%s-%s.%s.%s" % (self.sourceDir, targetrpm.name, targetrpm.version, targetrpm.release, targetrpm.arch, suffix) - # construct new sequence filename - dest = "%s/%s-%s-%s.%s.%s" % (self.destinationDir, targetrpm.name, targetrpm.version, targetrpm.release, targetrpm.arch, suffix) - # copy content usind read/write - fr=open(source,'r') - content=fr.readlines() - fr.close() - fw=open(dest,'w') - fw.writelines(content) - fw.close() - return dest - - - def fetchSequence(self, oldrpm, targetrpm): - if DEBUG: - print 'DEBUG: %s.fetchSequence(%s,%s)' % (self.__class__, oldrpm, targetrpm) - return self.__copyFile(targetrpm, SEQ_SUFFIX) - - def fetchDeltaRpm(self, oldrpm, targetrpm): - if DEBUG: - print 'DEBUG: %s.fetchDeltaRpm(%s,%s)' % (self.__class__, oldrpm, targetrpm) - return self.__copyFile(targetrpm, 'rpm') - -def getInstalled(targetrpm, sizelimit=0): - """retrieve description of installed version from rpm database""" - if DEBUG: - print 'DEBUG: %s.getInstalled(%s)' % (__name__, targetrpm) - import rpm - ts = rpm.TransactionSet() - # ts.setVSFlags(-1) - mi = ts.dbMatch('name', targetrpm.name) - oldrpm = None - count = 0 - for h in mi: - oldrpmtmp = RpmDescription( h['name'], h['version'], h['release'], h['arch'], h['epoch']) - size = h['size'] - #print "sizelimit: %d, size: %d" % (sizelimit, size) - if sizelimit > 0 and size > sizelimit: - raise Exception ("package %s bigger than limit (%d, %d)" % (targetrpm.name, size, sizelimit)) - # TODO: add __cmp__ to RpmDescription to determine most current installed - # does not matter too much - for reconstruction any installed version is good - oldrpm = oldrpmtmp - count+=1 - continue - if oldrpm: - if oldrpm > oldrpmtmp: - oldrpm = oldrpmtmp - else: - oldrpm = oldrpmtmp - # cleanup handles to free all rpmdb transactions - avoid db locking - del mi - del ts - if DEBUG: - print 'DEBUG: %s.getInstalled(%s): %s matches, using %s' % (__name__, targetrpm, count,oldrpm) - return oldrpm - -def getPackageFromDelta(cfg, rpmarray): - # method to be invoked within up2date - # return filename of regenerated newrpm - # - if DEBUG: - print 'DEBUG: %s.getPackageFromDelta(%s)' % (__name__, rpmarray) - sizelimit=0 - if cfg.has_key(SIZELIMIT): - sizelimit = cfg[SIZELIMIT] - # 1. retrieve relevant config from rhncfg - if cfg.has_key(STORAGE): - storageDir = cfg[STORAGE] - else: - storageDir = DEFAULT_STORAGE - # TODO: check if is directory - if not os.access(storageDir, os.F_OK): - if DEBUG: - print 'DEBUG: %s.getPackageFromDelta: mkdir(%s)' % (__name__, storageDir) - os.makedirs(storageDir) - if cfg.has_key(DELTA_STORAGE): - deltaStorage = cfg[DELTA_STORAGE] - else: - deltaStorage = DEFAULT_DELTA_STORAGE - # TODO: check if is directory - if not os.access(deltaStorage, os.F_OK): - if DEBUG: - print 'DEBUG: %s.getPackageFromDelta: mkdir(%s)' % (__name__, deltaStorage) - os.makedirs(deltaStorage) - if cfg.has_key(DELTA_URL): - deltaUrl = cfg[DELTA_URL] - else: - # without URL we can't do anything useful - raise exception and let up2date fall back to its own retrieval - raise "%s not configured" % DELTA_URL - oldRpms = None - if cfg.has_key(DELTA_OLDRPM_REPOSITORY): - oldRpms = cfg[DELTA_OLDRPM_REPOSITORY] - # use both config setting where old rpms could be and if they should be used - if cfg.has_key(DELTA_USE_OLDRPMS): - useOldRpms = cfg[DELTA_USE_OLDRPMS] - else: - # default is to NOT use old rpms - useOldRpms = 0 - # if old rpms should be used, check if oldRpms is set , warn otherwise - if useOldRpms: - if not oldRpms: - print "warning: configuration inconsistent: cannot use ols rpms without path specified, check %s" % DELTA_OLDRPM_REPOSITORY - useOldRpms = 0 - if cfg.has_key(DELTA_USE_SEQ): - useSeq = cfg[DELTA_USE_SEQ] - else: - # default is to NOT use sequence files - useSeq = 0 - # 2. determine old rpm description - targetrpm = RpmDescription(rpmarray[0], rpmarray[1], rpmarray[2], rpmarray[4], rpmarray[3]) - oldrpm = getInstalled(targetrpm, sizelimit) - - # raise exception if package is not installed - if not oldrpm: - raise Exception("%s is not installed" % targetrpm.name) - - # TODO: determine based on URL setting whih fetcher to use - fetcher = HttpFetcher(deltaUrl, deltaStorage) - #fetcher = TestFSFetcher('/tmp/deltasource', deltaStorage) - # wrapper takes only paths as constructor arguments, - # flags like useSeq or useOldRpms can be set on every method invocation - wrapper = DeltaRpmWrapper(storageDir, oldRpms) - # 3. ifSeq: - if useSeq: - # 3.1. download seq - seqfile = fetcher.fetchSequence(oldrpm, targetrpm) - if DEBUG: - print 'DEBUG: %s.getPackageFromDelta: received seq in %s' % (__name__, seqfile) - # 3.2 verify seq - wrapper.verifySequence(seqfile, oldrpm, useOldRpms) - # 4. download deltarpm - deltafile = fetcher.fetchDeltaRpm(oldrpm, targetrpm) - if DEBUG: - print 'DEBUG: %s.getPackageFromDelta: received rpm in %s' % (__name__, deltafile) - # 5. regenerate newrpm - newfile = wrapper.apply(oldrpm, targetrpm, deltafile, useOldRpms) - # output some statistics ;) - print "successfully reconstructed %s - %d bytes tranferred instead of %d" % (targetrpm, os.stat(deltafile).st_size, os.stat(newfile).st_size) - # done, cleanup - # 6. delete seq and deltarpm file if keepAfterInstall is not set - if cfg.has_key('keepAfterInstall') and cfg['keepAfterInstall']: - pass - else: - # let mkdir operation without try/except as failure would mean that something is really broken - # up2date would fallback to its retrieval and therefore not rely at all on this code ;) - if DEBUG: - print 'DEBUG: %s.getPackageFromDelta: rm(%s)' % (__name__, deltafile) - os.unlink(deltafile) - if useSeq: - if DEBUG: - print 'DEBUG: %s.getPackageFromDelta: rm(%s)' % (__name__, seqfile) - os.unlink(seqfile) - return newfile - -def getDrpmName(oldrpm, targetrpm): - """Get delta rpm name from old, new rpms, and suffix""" - dver = "_".join([targetrpm.version, oldrpm.version] ) - drel = "_".join([targetrpm.release ,oldrpm.release] ) - drpmName = '%s-%s-%s.%s' % (oldrpm.name, dver, drel, oldrpm.arch) - return drpmName - -if __name__ == '__main__': - import sys - arg = sys.argv[1] - newrpm = RpmDescription(arg,'1.0.6','1.4.1','i386') - old = getInstalled(newrpm, 0) - old = getInstalled(newrpm, 10000000) - - print old.rhnFileName() - #p = Process() - #p.run('find','/var/Satellite','-xdev') - #print p.getOutput() - #print p.returnCode() - print |