diff options
Diffstat (limited to 'deltarpm.py')
| -rw-r--r-- | deltarpm.py | 513 |
1 files changed, 513 insertions, 0 deletions
diff --git a/deltarpm.py b/deltarpm.py new file mode 100644 index 0000000..aac0c02 --- /dev/null +++ b/deltarpm.py @@ -0,0 +1,513 @@ +# author: Lars Herrmann <herrmann@redhat.com> +# Ahmed Kamal <email.ahmedkamal@googlemail.com> +# license: GPL (see COPYING file in distribution) +# +# this module provides a python wrapper around deltarpm tools written by suse +# +# TODO: catch exceptions wherever possible and raise useful ones ;) +# see TODO lines in methods + +MAKE='/usr/bin/makedeltarpm' +APPLY='/usr/bin/applydeltarpm' +SEQ_SUFFIX='seq' +SUFFIX='drpm' + +# constants for up2date configuration +# major flag to enable or disable use of delta rpms , used in patched up2date +USE_DELTA='useDeltaRpms' +# directory where to store generated rpms - depending on use up2date storage or temp storage for fetching +STORAGE='storageDir' +# directory where to store delta rpm files, normally temporarily +DELTA_STORAGE='deltaStorageDir' +# url to fetch delta rpm files from +DELTA_URL='deltaRpmURL' +# directory where local copies of oldrpms reside +DELTA_OLDRPM_REPOSITORY='deltaOldRpmRepository' +# flag indicates if downloading and verfying sequence files before downloading delta rpms - saves bandwidth +DELTA_USE_SEQ='deltaUseSequences' +# flag indicates if local copies of old rpms should be used +DELTA_USE_OLDRPMS='deltaUseOldRpms' +SIZELIMIT='deltaRpmSizeLimit' + +# default setting for STORAGE - same as with up2date +DEFAULT_STORAGE='/var/spool/up2date' +# default setting for DELTA_STORAGE +DEFAULT_DELTA_STORAGE='/var/spool/up2date/deltarpms' + +# enable or disable to see verbose messages on stdout +DEBUG=0 + +import popen2 +import string +import os +import glob + +class Process: + """wrapper class to execute programs and return exitcode and output (stdout and stderr combined)""" + def __init__(self): + self.__stdout=None + self.__returncode=None + self.__command=None + self.__args=None + + def run(self, command, *args): + self.__command=command + self.__args=args + cmdline=command+" "+string.join(args, " ") + if DEBUG: + print 'DEBUG: %s.%s: executing %s' % (self.__class__, 'run', cmdline) + pipe = popen2.Popen4(cmdline) + self.__stdout=pipe.fromchild.read() + retcode = pipe.wait() + if os.WIFEXITED(retcode): + self.__returncode = os.WEXITSTATUS(retcode) + else: + self.__returncode = retcode + # fallback to old implementation - works better ? + #stdoutp = os.popen(cmdline,'r',1) + #self.__stdout = stdoutp.read() + #retcode = stdoutp.close() + #if retcode is None: + # self.__returncode = 0 + #else: + # self.__returncode = retcode + + def getOutput(self): + return self.__stdout + + def returnCode(self): + return self.__returncode + +class RpmDescription: + """Wrapper class to encapsulate RPM attributes""" + + def __init__(self, name,version,release,arch, epoch=''): + """constructor: provide major attributes in correct order - epoch optional""" + self.name=name + self.version=version + self.release=release + self.epoch=epoch + self.arch=arch + if DEBUG: + print 'DEBUG: %s.%s: created: %s' % (self.__class__, '__init__', self) + + def rhnFileName(self): + """return file name in e:nvr.a notation""" + if self.epoch: + return "%s:%s-%s-%s.%s" % (self.epoch, self.name, self.version, self.release, self.arch) + else: + return "%s-%s-%s.%s" % (self.name, self.version, self.release, self.arch) + + def evr(self): + """return file name in e:vr notation as used in Satellite repository""" + if self.epoch: + return "%s:%s-%s" % (self.epoch, self.version, self.release) + else: + return "%s-%s" % (self.version, self.release) + + def satellitePath(self): + """return file path as used in Satellite repository - relative to /var/satellite""" + return "%s/%s/%s/%s.rpm" % (self.name, self.evr(), self.arch, self.fileName()) + + def fileName(self): + """return file name in nvr.a notation as used in up2date storageDir""" + return "%s-%s-%s.%s" % (self.name, self.version, self.release, self.arch) + + def __str__(self): + return self.rhnFileName() + +class DeltaRpmWrapper: + """wrapper around deltarpm binaries - implement methods for create, apply and verify delta rpms + - raises exceptions if exitcode of binaries was != 0""" + + def __init__(self, storageDir, oldRpmDir=None): + """constructor - params: storageDir=path of delta rpm storage, oldRpmDir = path of full rpm repository (optional)""" + self.storageDir = storageDir + self.oldRpmDir = oldRpmDir + if DEBUG: + print 'DEBUG: %s.%s: created: %s' % (self.__class__, '__init__', self) + + def __str__(self): + return "%s: storageDir=%s, oldRpmDir=%s" % (self.__class__, self.storageDir, self.oldRpmDir) + + def create(self, oldrpm, newrpm): + """wraps execution of makedeltarpm -s seqfile oldrpm newrpm deltarpm + constructs file names and paths based on given RpmDescription and instance settings for directories""" + if DEBUG: + print 'DEBUG: %s.create(%s,%s)' % (self.__class__,oldrpm, newrpm) + + # contruct filenames in satellite repository + oldrpmfile = "%s/%s" % (self.oldRpmDir, oldrpm.satellitePath()) + newrpmfile = "%s/%s" % (self.oldRpmDir, newrpm.satellitePath()) + + # check if file exists + # this is an ugly workaround, where the epoch is 0, but satellite + # stores it as 0:package, so we try it with epoch = string "0" + # we do glob.glob here, because satellite path could also contain + # shell wildcards + if not glob.glob(oldrpmfile): + if not oldrpm.epoch: + oldrpm.epoch = '0' + oldrpmfile = "%s/%s" % (self.oldRpmDir, oldrpm.satellitePath()) + if not glob.glob(newrpmfile): + if not newrpm.epoch: + newrpm.epoch = '0' + newrpmfile = "%s/%s" % (self.oldRpmDir, newrpm.satellitePath()) + + # construct filenames in deltarpm repository: + # /root/oldrpm/newrpm.{seq|rpm} + # with oldrpm|newrpm in e:nvr.a notation + deltarpm = newrpm.rhnFileName() + # files should go to /root/oldrpm + deltadir = "%s/%s" % (self.storageDir, oldrpm.rhnFileName()) + # TODO check if is a directory + if not os.access(deltadir, os.F_OK): + if DEBUG: + print 'DEBUG: %s.create: mkdir(%s)' % (__name__, deltadir) + os.makedirs(deltadir) + # filenames + deltarpmfile = "%s/%s" % (deltadir, deltarpm) + p=Process() + p.run(MAKE, '-s', "%s.%s" % (deltarpmfile,SEQ_SUFFIX), oldrpmfile, newrpmfile, "%s.rpm" % deltarpmfile) + # save output into logfile + logfile = "%s.log" % deltarpmfile + fd = open(logfile,'w') + fd.write(p.getOutput()) + print >> fd, "of: %s \nnf: %s" % (oldrpmfile, newrpmfile) + + fd.close() + if p.returnCode(): + raise Exception("%s.create: exitcode was %s - see %s" % (self.__class__,p.returnCode(), logfile)) + return deltarpmfile + + def apply(self, oldrpm, newrpm, deltarpmfile, useOldRpms = 0): + """wraps execution of applydeltarpm [-r oldrpm] deltarpm newrpm - + constructs file names and paths based on given RpmDescription and instance settings for directories""" + # args: RpmDescription + # TODO: test args for type == instance and __class__ == RpmDescription + # TODO: test without useOldRpms + if DEBUG: + print 'DEBUG: %s.apply(%s,%s,%s,%s)' % (self.__class__,oldrpm, newrpm, deltarpmfile, useOldRpms) + p=Process() + # targetrpm filename + newrpmfile = "%s/%s-%s-%s.%s.rpm" % (self.storageDir, newrpm.name, newrpm.version, newrpm.release, newrpm.arch) + if useOldRpms: + # TODO: check if self.oldRpmDir is set and exists ! + oldrpmfile = "%s/%s-%s-%s.%s.rpm" % (self.oldRpmDir, oldrpm.name, oldrpm.version, oldrpm.release, oldrpm.arch) + p.run(APPLY, '-r', oldrpmfile, deltarpmfile, newrpmfile) + else: + p.run(APPLY, deltarpmfile, newrpmfile) + if p.returnCode(): + # in case of error save output into logfile - will not be removed for further inspection + logfile = "%s.log" % deltarpmfile + fd = open(logfile,'w') + fd.write(p.getOutput()) + fd.close() + raise Exception("%s.apply(%s) exitcode was %d - see %s" % (self.__class__, newrpm, p.returnCode(), logfile)) + return newrpmfile + + def verifySequence(self, sequencefile, oldrpm = None, useOldRpms = 0): + """wraps execution of applydeltarpm [-r oldrpm] -s seqfilecontent - + constructs file names and paths based on given RpmDescription and instance settings for directories""" + if DEBUG: + print 'DEBUG: %s.verify(%s,%s,%s)' % (self.__class__,sequencefile, oldrpm, useOldRpms) + # read sequencefile + fd = open(sequencefile) + # TODO: is strip safe here ? could remove other chars than the linebreak + content = string.strip(string.join(fd.readlines())) + fd.close() + p = Process() + if useOldRpms: + oldrpmfile = "%s/%s-%s-%s.%s.rpm" % (self.oldRpmDir, oldrpm.name, oldrpm.version, oldrpm.release, oldrpm.arch) + p.run(APPLY, '-s', content, '-r', oldrpmfile) + else: + p.run(APPLY, '-s', content) + if p.returnCode(): + # in case of error save output into logfile - will not be removed for further inspection + logfile = "%s.log" % deltarpmfile + fd = open(logfile,'w') + fd.write(p.getOutput()) + fd.close() + raise Exception("could not verify sequence of delta rpm: %d - see %s" % (p.returnCode(), logfile)) +class Fetcher: + """ abstract class to be derived from classes implementing fetching seq and rpm files """ + + def fetchSequence(self, oldrpm, targetrpm): + pass + + def fetchDeltaRpm(self, oldrpm, targetrpm): + pass + +class HttpFetcher(Fetcher): + """ fetching seq and rpm files via http urls""" + + def __init__(self, deltaUrl, destinationDir): + """constructor - params: deltaUrl = webapp-url, destinationDir=path to store files""" + self.deltaUrl = deltaUrl + self.destinationDir = destinationDir + if DEBUG: + print 'DEBUG: %s.%s: created: %s' % (self.__class__, '__init__', self) + + def fetchSequence(self, oldrpm, targetrpm): + if DEBUG: + print 'DEBUG: %s.fetchSequence: : (%s,%s)' % (self.__class__, oldrpm, targetrpm) + return self.__fetchFile(oldrpm, targetrpm, SEQ_SUFFIX, 1) + + # The following method has been disabled by Fedora Infrastructure team, as + # we will not be using a server side web service, rather, delta rpms will be generated + # periodically, and client side, will simply download them if applicable + def __DISABLED__fetchFile(self, oldrpm, targetrpm, suffix, sequence=0): + """private method - uses private module to do the http request to rely on http return code""" + import httppost + data={} + data['oldname'] = oldrpm.name + data['oldversion'] = oldrpm.version + data['oldrelease'] = oldrpm.release + # avoid that httplib would send 'None' and not empty string + if oldrpm.epoch: + data['oldepoch'] = oldrpm.epoch + else: + data['oldepoch']='' + data['oldarch'] = oldrpm.arch + data['newname'] = targetrpm.name + data['newversion'] = targetrpm.version + data['newrelease'] = targetrpm.release + if targetrpm.epoch: + data['newepoch'] = targetrpm.epoch + else: + data['newepoch'] = '' + data['newarch'] = targetrpm.arch + if sequence: + data['sequence']='1' + + fd = httppost.send(self.deltaUrl, data, DEBUG) + content = fd.read() + fd.close() + dest = "%s/%s-%s-%s.%s.%s" % (self.destinationDir, targetrpm.name, targetrpm.version, targetrpm.release, targetrpm.arch, suffix) + fd=open(dest,'w') + fd.write(content) + fd.close + return dest + def __fetchFile(self, oldrpm, targetrpm, suffix, sequence=0): + """private method - uses private module to download delta rpms""" + import urllib2 + + if sequence: + data['sequence']='1' + + drpmName = getDrpmName(oldrpm, targetrpm) + fullUrl = '%s%s.%s' % (self.deltaUrl,drpmName,suffix) + if DEBUG: + print 'DEBUG: oldrpm: %s, newrpm: %s, suffix: %s' % (oldrpm, targetrpm, suffix) + print 'DEBUG: %s.__fetchFile: : (%s)' % (self.__class__, fullUrl) + try: + fd = urllib2.urlopen(fullUrl) + except IOError, e: + if hasattr(e, 'reason'): + raise Exception ("Failed to download delta rpm from URL %s, error: %s" % (fullUrl,e.reason)) + elif hasattr(e, 'code'): + raise Exception ("Failed to download delta rpm from URL %s, error: %s" % (fullUrl,e.code)) + else: + content = fd.read() + fd.close() + dest = "%s/%s-%s-%s.%s.%s" % (self.destinationDir, targetrpm.name, targetrpm.version, targetrpm.release, targetrpm.arch, suffix) + fd=open(dest,'w') + fd.write(content) + fd.close + return dest + + def fetchDeltaRpm(self, oldrpm, targetrpm): + if DEBUG: + print 'DEBUG: %s.fetchDeltaRpm: : (%s,%s)' % (self.__class__, oldrpm, targetrpm) + return self.__fetchFile(oldrpm, targetrpm, SUFFIX, 0) + + def __str__(self): + return "%s: deltaUrl=%s, destinationDir=%s" % (self.__class__, self.deltaUrl, self.destinationDir) + +class TestFSFetcher(Fetcher): + """ fetching seq and rpm files from local filesystem - uses NOT same directory structure as DeltaRpmWrapper.create""" + + def __init__(self, sourceDir, destinationDir): + self.sourceDir = sourceDir + self.destinationDir = destinationDir + if DEBUG: + print 'DEBUG: %s.%s: created: %s' % (self.__class__, '__init__', self) + + def __str__(self): + return "%s: sourceDir=%s, destinationDir=%s" % (self.__class__, self.sourceDir, self.destinationDir) + + def __copyFile(self, targetrpm, suffix): + source = "%s/%s-%s-%s.%s.%s" % (self.sourceDir, targetrpm.name, targetrpm.version, targetrpm.release, targetrpm.arch, suffix) + # construct new sequence filename + dest = "%s/%s-%s-%s.%s.%s" % (self.destinationDir, targetrpm.name, targetrpm.version, targetrpm.release, targetrpm.arch, suffix) + # copy content usind read/write + fr=open(source,'r') + content=fr.readlines() + fr.close() + fw=open(dest,'w') + fw.writelines(content) + fw.close() + return dest + + + def fetchSequence(self, oldrpm, targetrpm): + if DEBUG: + print 'DEBUG: %s.fetchSequence(%s,%s)' % (self.__class__, oldrpm, targetrpm) + return self.__copyFile(targetrpm, SEQ_SUFFIX) + + def fetchDeltaRpm(self, oldrpm, targetrpm): + if DEBUG: + print 'DEBUG: %s.fetchDeltaRpm(%s,%s)' % (self.__class__, oldrpm, targetrpm) + return self.__copyFile(targetrpm, 'rpm') + +def getInstalled(targetrpm, sizelimit=0): + """retrieve description of installed version from rpm database""" + if DEBUG: + print 'DEBUG: %s.getInstalled(%s)' % (__name__, targetrpm) + import rpm + ts = rpm.TransactionSet() + # ts.setVSFlags(-1) + mi = ts.dbMatch('name', targetrpm.name) + oldrpm = None + count = 0 + for h in mi: + oldrpmtmp = RpmDescription( h['name'], h['version'], h['release'], h['arch'], h['epoch']) + size = h['size'] + #print "sizelimit: %d, size: %d" % (sizelimit, size) + if sizelimit > 0 and size > sizelimit: + raise Exception ("package %s bigger than limit (%d, %d)" % (targetrpm.name, size, sizelimit)) + # TODO: add __cmp__ to RpmDescription to determine most current installed + # does not matter too much - for reconstruction any installed version is good + oldrpm = oldrpmtmp + count+=1 + continue + if oldrpm: + if oldrpm > oldrpmtmp: + oldrpm = oldrpmtmp + else: + oldrpm = oldrpmtmp + # cleanup handles to free all rpmdb transactions - avoid db locking + del mi + del ts + if DEBUG: + print 'DEBUG: %s.getInstalled(%s): %s matches, using %s' % (__name__, targetrpm, count,oldrpm) + return oldrpm + +def getPackageFromDelta(cfg, rpmarray): + # method to be invoked within up2date + # return filename of regenerated newrpm + # + if DEBUG: + print 'DEBUG: %s.getPackageFromDelta(%s)' % (__name__, rpmarray) + sizelimit=0 + if cfg.has_key(SIZELIMIT): + sizelimit = cfg[SIZELIMIT] + # 1. retrieve relevant config from rhncfg + if cfg.has_key(STORAGE): + storageDir = cfg[STORAGE] + else: + storageDir = DEFAULT_STORAGE + # TODO: check if is directory + if not os.access(storageDir, os.F_OK): + if DEBUG: + print 'DEBUG: %s.getPackageFromDelta: mkdir(%s)' % (__name__, storageDir) + os.makedirs(storageDir) + if cfg.has_key(DELTA_STORAGE): + deltaStorage = cfg[DELTA_STORAGE] + else: + deltaStorage = DEFAULT_DELTA_STORAGE + # TODO: check if is directory + if not os.access(deltaStorage, os.F_OK): + if DEBUG: + print 'DEBUG: %s.getPackageFromDelta: mkdir(%s)' % (__name__, deltaStorage) + os.makedirs(deltaStorage) + if cfg.has_key(DELTA_URL): + deltaUrl = cfg[DELTA_URL] + else: + # without URL we can't do anything useful - raise exception and let up2date fall back to its own retrieval + raise "%s not configured" % DELTA_URL + oldRpms = None + if cfg.has_key(DELTA_OLDRPM_REPOSITORY): + oldRpms = cfg[DELTA_OLDRPM_REPOSITORY] + # use both config setting where old rpms could be and if they should be used + if cfg.has_key(DELTA_USE_OLDRPMS): + useOldRpms = cfg[DELTA_USE_OLDRPMS] + else: + # default is to NOT use old rpms + useOldRpms = 0 + # if old rpms should be used, check if oldRpms is set , warn otherwise + if useOldRpms: + if not oldRpms: + print "warning: configuration inconsistent: cannot use ols rpms without path specified, check %s" % DELTA_OLDRPM_REPOSITORY + useOldRpms = 0 + if cfg.has_key(DELTA_USE_SEQ): + useSeq = cfg[DELTA_USE_SEQ] + else: + # default is to NOT use sequence files + useSeq = 0 + # 2. determine old rpm description + targetrpm = RpmDescription(rpmarray[0], rpmarray[1], rpmarray[2], rpmarray[4], rpmarray[3]) + oldrpm = getInstalled(targetrpm, sizelimit) + + # raise exception if package is not installed + if not oldrpm: + raise Exception("%s is not installed" % targetrpm.name) + + # TODO: determine based on URL setting whih fetcher to use + fetcher = HttpFetcher(deltaUrl, deltaStorage) + #fetcher = TestFSFetcher('/tmp/deltasource', deltaStorage) + # wrapper takes only paths as constructor arguments, + # flags like useSeq or useOldRpms can be set on every method invocation + wrapper = DeltaRpmWrapper(storageDir, oldRpms) + # 3. ifSeq: + if useSeq: + # 3.1. download seq + seqfile = fetcher.fetchSequence(oldrpm, targetrpm) + if DEBUG: + print 'DEBUG: %s.getPackageFromDelta: received seq in %s' % (__name__, seqfile) + # 3.2 verify seq + wrapper.verifySequence(seqfile, oldrpm, useOldRpms) + # 4. download deltarpm + deltafile = fetcher.fetchDeltaRpm(oldrpm, targetrpm) + if DEBUG: + print 'DEBUG: %s.getPackageFromDelta: received rpm in %s' % (__name__, deltafile) + # 5. regenerate newrpm + newfile = wrapper.apply(oldrpm, targetrpm, deltafile, useOldRpms) + # output some statistics ;) + print "successfully reconstructed %s - %d bytes tranferred instead of %d" % (targetrpm, os.stat(deltafile).st_size, os.stat(newfile).st_size) + # done, cleanup + # 6. delete seq and deltarpm file if keepAfterInstall is not set + if cfg.has_key('keepAfterInstall') and cfg['keepAfterInstall']: + pass + else: + # let mkdir operation without try/except as failure would mean that something is really broken + # up2date would fallback to its retrieval and therefore not rely at all on this code ;) + if DEBUG: + print 'DEBUG: %s.getPackageFromDelta: rm(%s)' % (__name__, deltafile) + os.unlink(deltafile) + if useSeq: + if DEBUG: + print 'DEBUG: %s.getPackageFromDelta: rm(%s)' % (__name__, seqfile) + os.unlink(seqfile) + return newfile + +def getDrpmName(oldrpm, targetrpm): + """Get delta rpm name from old, new rpms, and suffix""" + dver = "_".join([targetrpm.version, oldrpm.version] ) + drel = "_".join([targetrpm.release ,oldrpm.release] ) + drpmName = '%s-%s-%s.%s' % (oldrpm.name, dver, drel, oldrpm.arch) + return drpmName + +if __name__ == '__main__': + import sys + arg = sys.argv[1] + newrpm = RpmDescription(arg,'1.0.6','1.4.1','i386') + old = getInstalled(newrpm, 0) + old = getInstalled(newrpm, 10000000) + + print old.rhnFileName() + #p = Process() + #p.run('find','/var/Satellite','-xdev') + #print p.getOutput() + #print p.returnCode() + print |