path: root/deltarpm.py

# author: Lars Herrmann <herrmann@redhat.com>
#	Ahmed Kamal <email.ahmedkamal@googlemail.com>
# license: GPL (see COPYING file in distribution)
#
# this module provides a python wrapper around deltarpm tools written by suse
# 
# TODO: catch exceptions wherever possible and raise useful ones ;)
#	see TODO lines in methods

MAKE='/usr/bin/makedeltarpm'
APPLY='/usr/bin/applydeltarpm'
SEQ_SUFFIX='seq'
SUFFIX='drpm'

# constants for up2date configuration
# major flag to enable or disable use of delta rpms , used in patched up2date
USE_DELTA='useDeltaRpms'
# directory where to store generated rpms - depending on use up2date storage or temp storage for fetching
STORAGE='storageDir'
# directory where to store delta rpm files, normally temporarily
DELTA_STORAGE='deltaStorageDir'
# url to fetch delta rpm files from
DELTA_URL='deltaRpmURL'
# directory where local copies of oldrpms reside
DELTA_OLDRPM_REPOSITORY='deltaOldRpmRepository'
# flag indicates if downloading and verfying sequence files before downloading delta rpms - saves bandwidth
DELTA_USE_SEQ='deltaUseSequences'
# flag indicates if local copies of old rpms should be used
DELTA_USE_OLDRPMS='deltaUseOldRpms'
SIZELIMIT='deltaRpmSizeLimit'

# default setting for STORAGE - same as with up2date
DEFAULT_STORAGE='/var/spool/up2date'
# default setting for DELTA_STORAGE
DEFAULT_DELTA_STORAGE='/var/spool/up2date/deltarpms'

# enable or disable to see verbose messages on stdout
DEBUG=0

import popen2
import string
import os
import glob

class Process:
	"""wrapper class to execute programs and return exitcode and output (stdout and stderr combined)"""
	def __init__(self):
		self.__stdout=None
		self.__returncode=None
		self.__command=None
		self.__args=None

	def run(self, command, *args):
		self.__command=command
		self.__args=args
		cmdline=command+" "+string.join(args, " ")
		if DEBUG:
			print 'DEBUG: %s.%s: executing %s' % (self.__class__, 'run', cmdline)
		pipe = popen2.Popen4(cmdline)
		self.__stdout=pipe.fromchild.read()
		retcode = pipe.wait()
		if os.WIFEXITED(retcode):
			self.__returncode = os.WEXITSTATUS(retcode)
		else:
			self.__returncode = retcode
		# fallback to old implementation - works better ?
		#stdoutp = os.popen(cmdline,'r',1)
		#self.__stdout = stdoutp.read()
		#retcode = stdoutp.close()
		#if retcode is None:
		#	self.__returncode = 0
		#else:
		#	self.__returncode = retcode

	def getOutput(self):
		return self.__stdout

	def returnCode(self):
		return self.__returncode

class RpmDescription:
	"""Wrapper class to encapsulate RPM attributes"""

	def __init__(self, name,version,release,arch, epoch=''):
		"""constructor: provide major attributes in correct order - epoch optional"""
		self.name=name
		self.version=version
		self.release=release
		self.epoch=epoch
		self.arch=arch
		if DEBUG:
			print 'DEBUG: %s.%s: created: %s' % (self.__class__, '__init__', self)

	def rhnFileName(self):
		"""return file name in e:nvr.a notation"""
		if self.epoch:
			return "%s:%s-%s-%s.%s" % (self.epoch, self.name, self.version, self.release, self.arch)
		else:
			return "%s-%s-%s.%s" % (self.name, self.version, self.release, self.arch)

	def evr(self):
		"""return file name in e:vr notation as used in Satellite repository"""
		if self.epoch:
			return "%s:%s-%s" % (self.epoch, self.version, self.release)
		else:
			return "%s-%s" % (self.version, self.release)

	def satellitePath(self):
		"""return file path as used in Satellite repository - relative to /var/satellite"""
		return "%s/%s/%s/%s.rpm" % (self.name, self.evr(), self.arch, self.fileName())

	def fileName(self):
		"""return file name in nvr.a notation as used in up2date storageDir"""
		return "%s-%s-%s.%s" % (self.name, self.version, self.release, self.arch)

	def __str__(self):
		return self.rhnFileName()

class DeltaRpmWrapper:
	"""wrapper around deltarpm binaries - implement methods for create, apply and verify delta rpms
	- raises exceptions if exitcode of binaries was != 0"""
	
	def __init__(self, storageDir, oldRpmDir=None):
		"""constructor - params: storageDir=path of delta rpm storage, oldRpmDir = path of full rpm repository (optional)"""
		self.storageDir = storageDir
		self.oldRpmDir = oldRpmDir
		if DEBUG:
			print 'DEBUG: %s.%s: created: %s' % (self.__class__, '__init__', self)

	def __str__(self):
		return "%s: storageDir=%s, oldRpmDir=%s" % (self.__class__, self.storageDir, self.oldRpmDir)

	def create(self, oldrpm, newrpm):
		"""wraps execution of makedeltarpm -s seqfile oldrpm newrpm deltarpm
		constructs file names and paths based on given RpmDescription and instance settings for directories"""
		if DEBUG:
			print 'DEBUG: %s.create(%s,%s)' % (self.__class__,oldrpm, newrpm)

		# contruct filenames in satellite repository
		oldrpmfile = "%s/%s" % (self.oldRpmDir, oldrpm.satellitePath())
		newrpmfile = "%s/%s" % (self.oldRpmDir, newrpm.satellitePath())

		# check if file exists
		# this is an ugly workaround, where the epoch is 0, but satellite
		# stores it as 0:package, so we try it with epoch = string "0"
		# we do glob.glob here, because satellite path could also contain
		# shell wildcards
		if not glob.glob(oldrpmfile):
			if not oldrpm.epoch:
				oldrpm.epoch = '0'
				oldrpmfile = "%s/%s" % (self.oldRpmDir, oldrpm.satellitePath())
		if not glob.glob(newrpmfile):
			if not newrpm.epoch:
				newrpm.epoch = '0'
				newrpmfile = "%s/%s" % (self.oldRpmDir, newrpm.satellitePath())

		# construct filenames in deltarpm repository:
		#   /root/oldrpm/newrpm.{seq|rpm}
		# with oldrpm|newrpm in e:nvr.a notation
		deltarpm = newrpm.rhnFileName()
		# files should go to /root/oldrpm
		deltadir = "%s/%s" % (self.storageDir, oldrpm.rhnFileName())
		# TODO check if is a directory
		if not os.access(deltadir, os.F_OK):
			if DEBUG:
				print 'DEBUG: %s.create: mkdir(%s)' % (__name__, deltadir)
			os.makedirs(deltadir)
		# filenames 
		deltarpmfile = "%s/%s" % (deltadir, deltarpm)
		p=Process()
		p.run(MAKE, '-s', "%s.%s" % (deltarpmfile,SEQ_SUFFIX), oldrpmfile, newrpmfile, "%s.rpm" % deltarpmfile)
		# save output into logfile
		logfile = "%s.log" % deltarpmfile
		fd = open(logfile,'w')
		fd.write(p.getOutput())
		print >> fd,  "of: %s \nnf: %s" % (oldrpmfile, newrpmfile)

		fd.close()
		if p.returnCode():
			raise Exception("%s.create: exitcode was %s - see %s" % (self.__class__,p.returnCode(), logfile))
		return deltarpmfile

	def apply(self, oldrpm, newrpm, deltarpmfile, useOldRpms = 0):
		"""wraps execution of applydeltarpm [-r oldrpm] deltarpm newrpm -
		constructs file names and paths based on given RpmDescription and instance settings for directories"""
		# args: RpmDescription
		# TODO: test args for type == instance and __class__ == RpmDescription
		# TODO: test without useOldRpms
		if DEBUG:
			print 'DEBUG: %s.apply(%s,%s,%s,%s)' % (self.__class__,oldrpm, newrpm, deltarpmfile, useOldRpms)
		p=Process()
		# targetrpm filename
		newrpmfile = "%s/%s-%s-%s.%s.rpm" % (self.storageDir, newrpm.name, newrpm.version, newrpm.release, newrpm.arch)
		if useOldRpms:
			# TODO: check if self.oldRpmDir is set and exists !
			oldrpmfile = "%s/%s-%s-%s.%s.rpm" % (self.oldRpmDir, oldrpm.name, oldrpm.version, oldrpm.release, oldrpm.arch)
			p.run(APPLY, '-r', oldrpmfile, deltarpmfile, newrpmfile)
		else:
			p.run(APPLY, deltarpmfile, newrpmfile)
		if p.returnCode():
			# in case of error save output into logfile - will not be removed for further inspection
			logfile = "%s.log" % deltarpmfile
			fd = open(logfile,'w')
			fd.write(p.getOutput())
			fd.close()
			raise Exception("%s.apply(%s) exitcode was %d - see %s" % (self.__class__, newrpm, p.returnCode(), logfile))
		return newrpmfile

	def verifySequence(self, sequencefile, oldrpm = None, useOldRpms = 0):
		"""wraps execution of applydeltarpm [-r oldrpm] -s seqfilecontent -
		constructs file names and paths based on given RpmDescription and instance settings for directories"""
		if DEBUG:
			print 'DEBUG: %s.verify(%s,%s,%s)' % (self.__class__,sequencefile, oldrpm, useOldRpms)
		# read sequencefile
		fd = open(sequencefile)
		# TODO: is strip safe here ? could remove other chars than the linebreak
		content = string.strip(string.join(fd.readlines()))
		fd.close()
		p = Process()
		if useOldRpms:
			oldrpmfile = "%s/%s-%s-%s.%s.rpm" % (self.oldRpmDir, oldrpm.name, oldrpm.version, oldrpm.release, oldrpm.arch)
			p.run(APPLY, '-s', content, '-r', oldrpmfile)
		else:
			p.run(APPLY, '-s', content)
		if p.returnCode():
			# in case of error save output into logfile - will not be removed for further inspection
			logfile = "%s.log" % deltarpmfile
			fd = open(logfile,'w')
			fd.write(p.getOutput())
			fd.close()
			raise Exception("could not verify sequence of delta rpm: %d - see %s" % (p.returnCode(), logfile))
class Fetcher:
	""" abstract class to be derived from classes implementing fetching seq and rpm files """

	def fetchSequence(self, oldrpm, targetrpm):
		pass

	def fetchDeltaRpm(self, oldrpm, targetrpm):
		pass

class HttpFetcher(Fetcher):
	""" fetching seq and rpm files via http urls"""

	def __init__(self, deltaUrl, destinationDir):
		"""constructor - params: deltaUrl = webapp-url, destinationDir=path to store files"""
		self.deltaUrl = deltaUrl
		self.destinationDir = destinationDir
		if DEBUG:
			print 'DEBUG: %s.%s: created: %s' % (self.__class__, '__init__', self)

	def fetchSequence(self, oldrpm, targetrpm):
		if DEBUG:
			print 'DEBUG: %s.fetchSequence: : (%s,%s)' % (self.__class__, oldrpm, targetrpm)
		return self.__fetchFile(oldrpm, targetrpm, SEQ_SUFFIX, 1)

	# The following method has been disabled by Fedora Infrastructure team, as
	# we will not be using a server side web service, rather, delta rpms will be generated
	# periodically, and client side, will simply download them if applicable
	def __DISABLED__fetchFile(self, oldrpm, targetrpm, suffix, sequence=0):
		"""private method - uses private module to do the http request to rely on http return code"""
		import httppost
		data={}
		data['oldname'] =  oldrpm.name
		data['oldversion'] = oldrpm.version
		data['oldrelease'] = oldrpm.release
		# avoid that httplib would send 'None' and not empty string
		if oldrpm.epoch:
			data['oldepoch'] = oldrpm.epoch
		else:
			data['oldepoch']=''
		data['oldarch'] = oldrpm.arch
		data['newname'] = targetrpm.name
		data['newversion'] = targetrpm.version
		data['newrelease'] = targetrpm.release
		if targetrpm.epoch:
			data['newepoch'] = targetrpm.epoch
		else:
			data['newepoch'] = ''
		data['newarch'] = targetrpm.arch
		if sequence:
			data['sequence']='1'

		fd = httppost.send(self.deltaUrl, data, DEBUG)
		content = fd.read()
		fd.close()
		dest = "%s/%s-%s-%s.%s.%s" % (self.destinationDir, targetrpm.name, targetrpm.version, targetrpm.release, targetrpm.arch, suffix)
		fd=open(dest,'w')
		fd.write(content)
		fd.close
		return dest
	def __fetchFile(self, oldrpm, targetrpm, suffix, sequence=0):
		"""private method - uses private module to download delta rpms"""
		import urllib2
		
		if sequence:
			data['sequence']='1'
		
		drpmName = getDrpmName(oldrpm, targetrpm)
		fullUrl = '%s%s.%s' % (self.deltaUrl,drpmName,suffix)
		if DEBUG:
			print 'DEBUG: oldrpm: %s, newrpm: %s, suffix: %s' % (oldrpm, targetrpm, suffix)
			print 'DEBUG: %s.__fetchFile: : (%s)' % (self.__class__, fullUrl)
		try:
			fd = urllib2.urlopen(fullUrl)
		except IOError, e:
			if hasattr(e, 'reason'):
				raise Exception ("Failed to download delta rpm from URL %s, error: %s" % (fullUrl,e.reason))
			elif hasattr(e, 'code'):
				raise Exception ("Failed to download delta rpm from URL %s, error: %s" % (fullUrl,e.code))
		else:
			content = fd.read()
			fd.close()
			dest = "%s/%s-%s-%s.%s.%s" % (self.destinationDir, targetrpm.name, targetrpm.version, targetrpm.release, targetrpm.arch, suffix)
			fd=open(dest,'w')
			fd.write(content)
			fd.close
			return dest

	def fetchDeltaRpm(self, oldrpm, targetrpm):
		if DEBUG:
			print 'DEBUG: %s.fetchDeltaRpm: : (%s,%s)' % (self.__class__, oldrpm, targetrpm)
		return self.__fetchFile(oldrpm, targetrpm, SUFFIX, 0)

	def __str__(self):
		return "%s: deltaUrl=%s, destinationDir=%s" % (self.__class__, self.deltaUrl, self.destinationDir)

class TestFSFetcher(Fetcher):
	""" fetching seq and rpm files from local filesystem - uses NOT same directory structure as DeltaRpmWrapper.create"""

	def __init__(self, sourceDir, destinationDir):
		self.sourceDir = sourceDir
		self.destinationDir = destinationDir
		if DEBUG:
			print 'DEBUG: %s.%s: created: %s' % (self.__class__, '__init__', self)

	def __str__(self):
		return "%s: sourceDir=%s, destinationDir=%s" % (self.__class__, self.sourceDir, self.destinationDir)

	def __copyFile(self, targetrpm, suffix):
		source = "%s/%s-%s-%s.%s.%s" % (self.sourceDir, targetrpm.name, targetrpm.version, targetrpm.release, targetrpm.arch, suffix)
		# construct new sequence filename
		dest = "%s/%s-%s-%s.%s.%s" % (self.destinationDir, targetrpm.name, targetrpm.version, targetrpm.release, targetrpm.arch, suffix)
		# copy content usind read/write
		fr=open(source,'r')
		content=fr.readlines()
		fr.close()
		fw=open(dest,'w')
		fw.writelines(content)
		fw.close()
		return dest


	def fetchSequence(self, oldrpm, targetrpm):
		if DEBUG:
			print 'DEBUG: %s.fetchSequence(%s,%s)' % (self.__class__, oldrpm, targetrpm)
		return self.__copyFile(targetrpm, SEQ_SUFFIX)

	def fetchDeltaRpm(self, oldrpm, targetrpm):
		if DEBUG:
			print 'DEBUG: %s.fetchDeltaRpm(%s,%s)' % (self.__class__, oldrpm, targetrpm)
		return self.__copyFile(targetrpm, 'rpm')

def getInstalled(targetrpm, sizelimit=0):
	"""retrieve description of installed version from rpm database"""
	if DEBUG:
		print 'DEBUG: %s.getInstalled(%s)' % (__name__, targetrpm)
	import rpm
	ts = rpm.TransactionSet()
	# ts.setVSFlags(-1)
	mi = ts.dbMatch('name', targetrpm.name)
	oldrpm = None
	count = 0
	for h in mi:
		oldrpmtmp = RpmDescription( h['name'], h['version'], h['release'], h['arch'], h['epoch'])
		size = h['size']
		#print "sizelimit: %d, size: %d" % (sizelimit, size)
		if sizelimit > 0 and size > sizelimit:
			raise Exception ("package %s bigger than limit (%d, %d)" % (targetrpm.name, size, sizelimit))
		# TODO: add __cmp__ to RpmDescription to determine most current installed
		# does not matter too much - for reconstruction any installed version is good
		oldrpm = oldrpmtmp
		count+=1
		continue
		if oldrpm:
			if oldrpm > oldrpmtmp:
				oldrpm = oldrpmtmp
			else:
				oldrpm = oldrpmtmp
	# cleanup handles to free all rpmdb transactions - avoid db locking
	del mi
	del ts
	if DEBUG:
		print 'DEBUG: %s.getInstalled(%s): %s matches, using %s' % (__name__, targetrpm, count,oldrpm)
	return oldrpm
		
def getPackageFromDelta(cfg, rpmarray):
	# method to be invoked within up2date
	# return filename of regenerated newrpm
	#
	if DEBUG:
		print 'DEBUG: %s.getPackageFromDelta(%s)' % (__name__, rpmarray)
	sizelimit=0
	if cfg.has_key(SIZELIMIT):
		sizelimit = cfg[SIZELIMIT]
	# 1. retrieve relevant config from rhncfg
	if cfg.has_key(STORAGE):
		storageDir = cfg[STORAGE]
	else:
		storageDir = DEFAULT_STORAGE
	# TODO: check if is directory
	if not os.access(storageDir, os.F_OK):
		if DEBUG:
			print 'DEBUG: %s.getPackageFromDelta: mkdir(%s)' % (__name__, storageDir)
		os.makedirs(storageDir)
	if cfg.has_key(DELTA_STORAGE):
		deltaStorage = cfg[DELTA_STORAGE]
	else:
		deltaStorage = DEFAULT_DELTA_STORAGE
	# TODO: check if is directory
	if not os.access(deltaStorage, os.F_OK):
		if DEBUG:
			print 'DEBUG: %s.getPackageFromDelta: mkdir(%s)' % (__name__, deltaStorage)
		os.makedirs(deltaStorage)
	if cfg.has_key(DELTA_URL):
		deltaUrl = cfg[DELTA_URL]
	else:
		# without URL we can't do anything useful - raise exception and let up2date fall back to its own retrieval
		raise "%s not configured" % DELTA_URL
	oldRpms = None
	if cfg.has_key(DELTA_OLDRPM_REPOSITORY):
		oldRpms = cfg[DELTA_OLDRPM_REPOSITORY]
	# use both config setting where old rpms could be and if they should be used
	if cfg.has_key(DELTA_USE_OLDRPMS):
		useOldRpms = cfg[DELTA_USE_OLDRPMS]
	else:
		# default is to NOT use old rpms
		useOldRpms = 0
	# if old rpms should be used, check if oldRpms is set , warn otherwise
	if useOldRpms:
		if not oldRpms:
			print "warning: configuration inconsistent: cannot use ols rpms without path specified, check %s" % DELTA_OLDRPM_REPOSITORY
			useOldRpms = 0
	if cfg.has_key(DELTA_USE_SEQ):
		useSeq = cfg[DELTA_USE_SEQ] 
	else:
		# default is to NOT use sequence files
		useSeq = 0
	# 2. determine old rpm description
	targetrpm = RpmDescription(rpmarray[0], rpmarray[1], rpmarray[2], rpmarray[4], rpmarray[3])
	oldrpm = getInstalled(targetrpm, sizelimit)
	
	# raise exception if package is not installed
	if not oldrpm:
		raise Exception("%s is not installed" % targetrpm.name)

	# TODO: determine based on URL setting whih fetcher to use
	fetcher = HttpFetcher(deltaUrl, deltaStorage)
	#fetcher = TestFSFetcher('/tmp/deltasource', deltaStorage)
	# wrapper takes only paths as constructor arguments, 
	# flags like useSeq or useOldRpms can be set on every method invocation
	wrapper = DeltaRpmWrapper(storageDir, oldRpms)
	# 3. ifSeq:
	if useSeq:
		# 3.1. download seq
		seqfile = fetcher.fetchSequence(oldrpm, targetrpm)
		if DEBUG:
			print 'DEBUG: %s.getPackageFromDelta: received seq in %s' % (__name__, seqfile)
		# 3.2  verify seq
		wrapper.verifySequence(seqfile, oldrpm, useOldRpms)
	# 4. download deltarpm
	deltafile = fetcher.fetchDeltaRpm(oldrpm, targetrpm)
	if DEBUG:
		print 'DEBUG: %s.getPackageFromDelta: received rpm in %s' % (__name__, deltafile)
	# 5. regenerate newrpm
	newfile = wrapper.apply(oldrpm, targetrpm, deltafile, useOldRpms)
        # output some statistics ;)
        print "successfully reconstructed %s - %d bytes tranferred instead of %d" % (targetrpm, os.stat(deltafile).st_size, os.stat(newfile).st_size)
	# done, cleanup
	# 6. delete seq and deltarpm file if keepAfterInstall is not set
	if cfg.has_key('keepAfterInstall') and cfg['keepAfterInstall']:
		pass
	else:
		# let mkdir operation without try/except as failure would mean that something is really broken
		# up2date would fallback to its retrieval and therefore not rely at all on this code ;)
		if DEBUG:
			print 'DEBUG: %s.getPackageFromDelta: rm(%s)' % (__name__, deltafile)
		os.unlink(deltafile)
		if useSeq:
			if DEBUG:
				print 'DEBUG: %s.getPackageFromDelta: rm(%s)' % (__name__, seqfile)
			os.unlink(seqfile)
	return newfile

def getDrpmName(oldrpm, targetrpm):
	"""Get delta rpm name from old, new rpms, and suffix"""
	dver = "_".join([targetrpm.version, oldrpm.version] )
	drel = "_".join([targetrpm.release ,oldrpm.release] )
	drpmName =  '%s-%s-%s.%s' % (oldrpm.name, dver, drel, oldrpm.arch)
	return drpmName

if __name__ == '__main__':
	import sys
	arg = sys.argv[1]
	newrpm = RpmDescription(arg,'1.0.6','1.4.1','i386')
	old = getInstalled(newrpm, 0)
	old = getInstalled(newrpm, 10000000)

	print old.rhnFileName()
	#p = Process()
	#p.run('find','/var/Satellite','-xdev')
	#print p.getOutput()
	#print p.returnCode()
	print