diff options
| -rw-r--r-- | createprestorepo/ChangeLog | 2 | ||||
| -rw-r--r-- | presto-utils/COPYING (renamed from createprestorepo/COPYING) | 0 | ||||
| -rw-r--r-- | presto-utils/ChangeLog | 6 | ||||
| -rw-r--r-- | presto-utils/Makefile (renamed from createprestorepo/Makefile) | 2 | ||||
| -rw-r--r-- | presto-utils/README (renamed from createprestorepo/README) | 6 | ||||
| -rwxr-xr-x | presto-utils/createdeltarpms (renamed from createprestorepo/makedeltarepo) | 0 | ||||
| -rwxr-xr-x | presto-utils/createprestorepo (renamed from createprestorepo/createprestorepo) | 0 | ||||
| -rw-r--r-- | presto-utils/dumpMetadata.py (renamed from createprestorepo/dumpMetadata.py) | 1 | ||||
| -rwxr-xr-x | presto-utils/gendeltarpms.py (renamed from createprestorepo/gendeltarpms.py) | 170 | ||||
| -rwxr-xr-x | presto-utils/genprestometadata.py (renamed from createprestorepo/genprestometadata.py) | 104 | ||||
| -rw-r--r-- | presto-utils/packagelist.py (renamed from createprestorepo/packagelist.py) | 3 |
11 files changed, 229 insertions, 65 deletions
diff --git a/createprestorepo/ChangeLog b/createprestorepo/ChangeLog deleted file mode 100644 index 2de29c0..0000000 --- a/createprestorepo/ChangeLog +++ /dev/null @@ -1,2 +0,0 @@ -* Tue May 7 2007 Jonathan Dieter <jdieter@gmail.com> - 0.1.0 - - Initial release diff --git a/createprestorepo/COPYING b/presto-utils/COPYING index e77696a..e77696a 100644 --- a/createprestorepo/COPYING +++ b/presto-utils/COPYING diff --git a/presto-utils/ChangeLog b/presto-utils/ChangeLog new file mode 100644 index 0000000..1640457 --- /dev/null +++ b/presto-utils/ChangeLog @@ -0,0 +1,6 @@ +* Tue Jun 19 2007 Jonathan Dieter <jdieter@gmail.com> - 0.2.0 + - Now works with createrepo and modifyrepo + - Many bugfixes + +* Tue May 7 2007 Jonathan Dieter <jdieter@gmail.com> - 0.1.0 + - Initial release diff --git a/createprestorepo/Makefile b/presto-utils/Makefile index 51228f8..9e8c6eb 100644 --- a/createprestorepo/Makefile +++ b/presto-utils/Makefile @@ -3,7 +3,7 @@ clean: install: install -m 755 createprestorepo $(DESTDIR)/usr/bin/ - install -m 755 makedeltarepo $(DESTDIR)/usr/bin/ + install -m 755 createdeltarpms $(DESTDIR)/usr/bin/ mkdir -p $(DESTDIR)/usr/share/createprestorepo install -m 755 genprestometadata.py $(DESTDIR)/usr/share/createprestorepo install -m 644 dumpMetadata.py $(DESTDIR)/usr/share/createprestorepo diff --git a/createprestorepo/README b/presto-utils/README index 010fdd3..a81187a 100644 --- a/createprestorepo/README +++ b/presto-utils/README @@ -10,9 +10,9 @@ Installation: 2- Run 'make install' Running: -1- First run 'makedeltarepo <repo dir> <drpm dir>' where <repo dir> is the base - directory for your repository and <drpm dir> is the subdirectory you want to - create the deltarpms into +1- First run 'createdeltarpms <repo dir> <drpm dir>' where <repo dir> is the + base directory for your repository and <drpm dir> is the subdirectory you + want to create the deltarpms into 2- Run 'createprestorepo <repo dir>' where <repo dir> is the base directory for your repository. diff --git a/createprestorepo/makedeltarepo b/presto-utils/createdeltarpms index 99bedcc..99bedcc 100755 --- a/createprestorepo/makedeltarepo +++ b/presto-utils/createdeltarpms diff --git a/createprestorepo/createprestorepo b/presto-utils/createprestorepo index 5334a80..5334a80 100755 --- a/createprestorepo/createprestorepo +++ b/presto-utils/createprestorepo diff --git a/createprestorepo/dumpMetadata.py b/presto-utils/dumpMetadata.py index 6969400..0ec2c20 100644 --- a/createprestorepo/dumpMetadata.py +++ b/presto-utils/dumpMetadata.py @@ -494,3 +494,4 @@ def repoXML(node, cmds): timestamp = data.newChild(None, 'timestamp', str(timestamp)) unchecksum = data.newChild(None, 'open-checksum', uncsum) unchecksum.newProp('type', sumtype) + diff --git a/createprestorepo/gendeltarpms.py b/presto-utils/gendeltarpms.py index 26d795b..7db9d4f 100755 --- a/createprestorepo/gendeltarpms.py +++ b/presto-utils/gendeltarpms.py @@ -15,11 +15,12 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA -import errno, os, sys, gzip +import errno, os, sys import fnmatch, re import rpmUtils.transaction, rpmUtils.miscutils -import commands, libxml2 +import commands import string +import getopt from dumpMetadata import byteranges from packagelist import RpmItem, DrpmItem, PackageList @@ -27,9 +28,36 @@ import gzip from zlib import error as zlibError from gzip import write32u, FNAME - -DRPMWORTHKEEPINGTHRESH=0.7 +DRPMWORTHKEEPINGTHRESH=0.5 DEBUG=0 +__version__ = "0.2.0" + +def errorprint(stuff): + print >> sys.stderr, stuff + +def _(args): + """Stub function for translation""" + return args + +def usage(retval=1): + print _(""" + createdeltarpms [options] directory-of-packages directory-of-deltarpms + + Options: + -d, --dist-update <dir> = optional directory containing old distribution + to create an update from + -c, --count <number> = optional maximum number of deltarpms to create + per package. Default is maximum available + -n, --no-first = Don't create deltarpm against first rpm if number exceeds + count. Useful if you are running a continually-updated + distribution rather than one with set release cycles. + -q, --quiet = run quietly + -v, --verbose = run verbosely + -h, --help = show this help + -V, --version = output version + """) + + sys.exit(retval) def getDeltaNevr(filename): """Get nevr for src rpm of a deltarpm. Return a tuple of (n, e, v, r)""" @@ -91,11 +119,7 @@ def getFileList(path, ext, filelist): extlen = len(ext) totalpath = os.path.normpath(path) - try: - dir_list = os.listdir(totalpath) - except OSError, e: - errorprint(_('Error accessing directory %s, %s') % (totalpath, e)) - sys.exit(1) + dir_list = os.listdir(totalpath) for d in dir_list: if os.path.isdir(totalpath + '/' + d): @@ -105,7 +129,7 @@ def getFileList(path, ext, filelist): return filelist -def createPrestoRepo(base_dir, drpm_dir, dst_dir=None): +def createPrestoRepo(base_dir, drpm_dir, dst_dir=None, DrpmsPerPackage=0, DoFirst=True): ts = rpmUtils.transaction.initReadOnlyTransaction() changed = False @@ -114,11 +138,21 @@ def createPrestoRepo(base_dir, drpm_dir, dst_dir=None): print 'Using base dir: %s' % base_dir print 'Using destination dir: %s' % drpm_dir - rpm_files = getFileList(base_dir, ".rpm", []) + try: + rpm_files = getFileList(base_dir, ".rpm", []) + except OSError, e: + errorprint(_("Error: Unable to find directory %s: %s" % (base_dir, e))) + return False if dst_dir != None: - dst_rpm_files = getFileList(dst_dir, ".rpm", []) - drpm_files = getFileList(drpm_dir, ".drpm", []) - + try: + dst_rpm_files = getFileList(dst_dir, ".rpm", []) + except OSError, e: + errorprint(_("Error: Unable to find directory %s: %s" % (dst_dir, e))) + return False + try: + drpm_files = getFileList(drpm_dir, ".drpm", []) + except: + drpm_files = [] if not len(rpm_files): print ' Nothing found.' return changed @@ -192,25 +226,99 @@ def createPrestoRepo(base_dir, drpm_dir, dst_dir=None): rpm_list.dump() # Build deltarpms - if dst_dir != None: - rpm_list.makeDeltaRpms(DrpmsPerPackage=1, DoFirst=False) + rpm_list.makeDeltaRpms(DrpmsPerPackage, DoFirst) + return True + +def parseArgs(args): + """ + Parse the command line args return a commands dict and directory. + Sanity check all the things being passed in. + """ + cmds = {} + cmds['quiet'] = 0 + cmds['verbose'] = 0 + cmds['dist-update'] = None + cmds['count'] = None + cmds['do-first'] = 1 + try: + gopts, argsleft = getopt.getopt(args, 'hqvVd:c:n', ['help', 'quiet', 'verbose', 'version', 'dist-update=', 'count=', 'no-first']) + except getopt.error, e: + errorprint(_('Options Error: %s.') % e) + usage() + + try: + for arg,a in gopts: + if arg in ['-h','--help']: + usage(retval=0) + elif arg in ['-V', '--version']: + print '%s' % __version__ + sys.exit(0) + except ValueError, e: + errorprint(_('Options Error: %s') % e) + usage() + + + # make sure our dir makes sense before we continue + if len(argsleft) > 2: + errorprint(_('Error: You may only specify one rpm directory and one deltarpm directory.')) + usage() + elif len(argsleft) == 0: + errorprint(_('Error: Must specify an rpm directory to index and a destination deltarpm directory.')) + usage() else: - rpm_list.makeDeltaRpms(DrpmsPerPackage=5) + directories = argsleft + + try: + for arg,a in gopts: + if arg in ['-v', '--verbose']: + cmds['verbose'] = 1 + elif arg in ["-q", '--quiet']: + cmds['quiet'] = 1 + elif arg in ["-n", '--no-first']: + cmds['do-first'] = 0 + elif arg in ['-d', '--dist-update']: + if cmds['dist-update'] is not None: + errorprint(_('Error: Only one directory for distribution updates allowed.')) + usage() + else: + cmds['dist-update'] = a + elif arg in ['-c', '--count']: + if cmds['count'] is not None: + errorprint(_('Error: Only one count allowed.')) + usage() + else: + cmds['count'] = int(a) + + except ValueError, e: + errorprint(_('Options Error: %s') % e) + usage() + + rpm_directory = directories[0] + drpm_directory = directories[1] + + rpm_directory = os.path.normpath(rpm_directory) + drpm_directory = os.path.normpath(drpm_directory) + + # Fixup first directory + directories[0] = rpm_directory + directories[1] = drpm_directory + + # Set count + if cmds['count'] == None: + cmds['count'] = 0 + + return cmds, directories if __name__ == '__main__': - if len(sys.argv) < 2 or (sys.argv[1] == "-d" and len(sys.argv) < 4): - print 'Usage: %s <bin_rpm_dir> <delta_rpm_dir>' % os.path.basename(sys.argv[0]) - print ' %s -d <old_dist_dir> <new_dist_dir> <delta_rpm_dir>' % os.path.basename(sys.argv[0]) - sys.exit(errno.EINVAL) - if sys.argv[1] != "-d": - bin_rpm_path = sys.argv[1] - delta_rpm_path = sys.argv[2] - createPrestoRepo(bin_rpm_path, delta_rpm_path) + cmds, directories = parseArgs(sys.argv[1:]) + if cmds['dist-update'] != None: + if createPrestoRepo(cmds['dist-update'], directories[1], directories[0], cmds['count'], cmds['do-first']): + sys.exit(0) + else: + sys.exit(1) else: - src_bin_rpm_path = sys.argv[2] - dst_bin_rpm_path = sys.argv[3] - delta_rpm_path = sys.argv[4] - createPrestoRepo(src_bin_rpm_path, delta_rpm_path, dst_bin_rpm_path) - - sys.exit(0) + if createPrestoRepo(directories[0], directories[1], None, cmds['count'], cmds['do-first']): + sys.exit(0) + else: + sys.exit(1) diff --git a/createprestorepo/genprestometadata.py b/presto-utils/genprestometadata.py index bbce09d..6788ffc 100755 --- a/createprestorepo/genprestometadata.py +++ b/presto-utils/genprestometadata.py @@ -27,10 +27,12 @@ import libxml2 import string import fnmatch import urlgrabber +import commands +import shutil import dumpMetadata from dumpMetadata import _gzipOpen -__version__ = '0.4.3' +__version__ = '0.2.0' def errorprint(stuff): print >> sys.stderr, stuff @@ -47,6 +49,7 @@ def usage(retval=1): -u, --baseurl <url> = optional base url location for all files -o, --outputdir <dir> = optional directory to output to -x, --exclude = files globs to exclude, can be specified multiple times + -m, --merge = merge with repository created by createrepo -q, --quiet = run quietly -v, --verbose = run verbosely -c, --cachedir <dir> = specify which dir to use for the checksum cache @@ -128,7 +131,10 @@ class MetaDataGenerator: self.formatns = self.prestoroot.newNs('http://linux.duke.edu/metadata/rpm', 'rpm') self.prestoroot.setNs(basens) prestofilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestofile']) - self.prestofile = _gzipOpen(prestofilepath, 'w') + if self.cmds['merge'] == 1: + self.prestofile = open(prestofilepath, 'w') + else: + self.prestofile = _gzipOpen(prestofilepath, 'w') self.prestofile.write('<?xml version="1.0" encoding="UTF-8"?>\n') self.prestofile.write('<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">\n' % self.pkgcount) @@ -173,26 +179,37 @@ class MetaDataGenerator: self.prestodoc.freeDoc() def doRepoMetadata(self): - """wrapper to generate the prestomd.xml file that stores the info on the other files""" - repodoc = libxml2.newDoc("1.0") - reporoot = repodoc.newChild(None, "repomd", None) - repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None) - reporoot.setNs(repons) - repofilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestomdfile']) + if self.cmds['prestomdfile'] == None: # Merge + shutil.copyfile(os.path.join(self.cmds['outputdir'], self.cmds['finaldir'], "repomd.xml"), os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], "repomd.xml")) + command = "/usr/bin/modifyrepo %s %s" % (os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestofile']), os.path.join(self.cmds['outputdir'], self.cmds['tempdir'])) + (code, out) = commands.getstatusoutput(command) + if code: + errorprint(_('\nAn error occurred trying to run modifyrepo: %s') % e) + sys.exit(1) + os.unlink(os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestofile'])) + self.cmds['prestofile'] = "%s.gz" % self.cmds['prestofile'] + self.cmds['prestomdfile'] = "repomd.xml" + else: + """wrapper to generate the prestomd.xml file that stores the info on the other files""" + repodoc = libxml2.newDoc("1.0") + reporoot = repodoc.newChild(None, "repomd", None) + repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None) + reporoot.setNs(repons) + repofilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestomdfile']) - try: - dumpMetadata.repoXML(reporoot, self.cmds) - except dumpMetadata.MDError, e: - errorprint(_('Error generating repo xml file: %s') % e) - sys.exit(1) + try: + dumpMetadata.repoXML(reporoot, self.cmds) + except dumpMetadata.MDError, e: + errorprint(_('Error generating repo xml file: %s') % e) + sys.exit(1) - try: - repodoc.saveFormatFileEnc(repofilepath, 'UTF-8', 1) - except: - errorprint(_('Error saving temp file for rep xml: %s') % repofilepath) - sys.exit(1) + try: + repodoc.saveFormatFileEnc(repofilepath, 'UTF-8', 1) + except: + errorprint(_('Error saving temp file for rep xml: %s') % repofilepath) + sys.exit(1) - del repodoc + del repodoc class SplitMetaDataGenerator(MetaDataGenerator): @@ -270,6 +287,7 @@ def parseArgs(args): cmds['excludes'] = [] cmds['baseurl'] = None cmds['sumtype'] = 'sha' + cmds['merge'] = 0 cmds['pretty'] = 0 cmds['cachedir'] = None cmds['basedir'] = os.getcwd() @@ -280,7 +298,7 @@ def parseArgs(args): cmds['dir-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*'] try: - gopts, argsleft = getopt.getopt(args, 'phqVvs:x:u:c:o:', ['help', 'exclude=', + gopts, argsleft = getopt.getopt(args, 'phqVvms:x:u:c:o:', ['help', 'exclude=', 'quiet', 'verbose', 'cachedir=', 'basedir=', 'baseurl=', 'checksum=', 'version', 'pretty', 'split', 'outputdir=']) @@ -328,6 +346,8 @@ def parseArgs(args): cmds['excludes'].append(a) elif arg in ['-p', '--pretty']: cmds['pretty'] = 1 + elif arg in ['-m', '--merge']: + cmds['merge'] = 1 elif arg in ['-c', '--cachedir']: cmds['cache'] = True cmds['cachedir'] = a @@ -363,8 +383,12 @@ def parseArgs(args): cmds['cachedir'] = a #setup some defaults - cmds['prestofile'] = 'presto.xml.gz' - cmds['prestomdfile'] = 'prestomd.xml' + if cmds['merge'] == 1: + cmds['prestofile'] = 'deltas.xml' + cmds['prestomdfile'] = None + else: + cmds['prestofile'] = 'presto.xml.gz' + cmds['prestomdfile'] = 'prestomd.xml' cmds['tempdir'] = '.repodata' cmds['finaldir'] = 'repodata' cmds['olddir'] = '.olddata' @@ -405,11 +429,12 @@ def main(args): # make sure we can write to where we want to write to: for direc in ['tempdir', 'finaldir']: for file in ['prestofile', 'prestomdfile']: - filepath = os.path.join(cmds['outputdir'], cmds[direc], cmds[file]) - if os.path.exists(filepath): - if not os.access(filepath, os.W_OK): - errorprint(_('error in must be able to write to metadata files:\n -> %s') % filepath) - usage() + if cmds[file] != None: + filepath = os.path.join(cmds['outputdir'], cmds[direc], cmds[file]) + if os.path.exists(filepath): + if not os.access(filepath, os.W_OK): + errorprint(_('error in must be able to write to metadata files:\n -> %s') % filepath) + usage() if cmds['split']: cmds['basedir'] = oldbase @@ -454,6 +479,31 @@ def main(args): sys.exit(1) + # Move everything else back from olddir (eg. repoview files) + olddir = os.path.join(cmds['outputdir'], cmds['olddir']) + finaldir = os.path.join(cmds['outputdir'], cmds['finaldir']) + for file in os.listdir(olddir): + oldfile = os.path.join(olddir, file) + finalfile = os.path.join(finaldir, file) + if os.path.exists(finalfile): + # Hmph? Just leave it alone, then. + try: + if os.path.isdir(oldfile): + shutil.rmtree(oldfile) + else: + os.remove(oldfile) + except OSError, e: + errorprint(_('Could not remove old non-metadata file: %s') % oldfile) + errorprint(_('Error was %s') % e) + sys.exit(1) + else: + try: + os.rename(oldfile, finalfile) + except OSError, e: + errorprint(_('Could not restore old non-metadata file: %s -> %s') % (oldfile, finalfile)) + errorprint(_('Error was %s') % e) + sys.exit(1) + #XXX: fix to remove tree as we mung basedir try: os.rmdir(os.path.join(cmds['outputdir'], cmds['olddir'])) diff --git a/createprestorepo/packagelist.py b/presto-utils/packagelist.py index c057f3f..e9019f9 100644 --- a/createprestorepo/packagelist.py +++ b/presto-utils/packagelist.py @@ -272,7 +272,7 @@ class PackageList: return False - def makeDeltaRpms(self, DrpmsPerPackage=0, OnlyLastPackage=True, DoFirst=True): + def makeDeltaRpms(self, DrpmsPerPackage=0, DoFirst=True, OnlyLastPackage=True): """Create deltarpms for packages. Attributes: OnlyLastPackage (Boolean): Only create deltarpms to point to last destination package @@ -383,3 +383,4 @@ class PackageList: if drpmsize > self.threshold * newsize: return False return True + |