summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJonathan Dieter <jdieter@gmail.com>2007-06-19 20:58:07 +0300
committerJonathan Dieter <jdieter@gmail.com>2007-06-19 20:58:07 +0300
commitdce0600bc64c793ba6e8f67c56c286d8d97e7c4c (patch)
tree71c559e031b3c10ba56a187e0a017f09d4d25137
parent93b2295180471308e969640472bdc601d1f10015 (diff)
downloadpresto-dce0600bc64c793ba6e8f67c56c286d8d97e7c4c.zip
presto-dce0600bc64c793ba6e8f67c56c286d8d97e7c4c.tar.gz
presto-dce0600bc64c793ba6e8f67c56c286d8d97e7c4c.tar.xz
Many bugfixes and a few enhancements
Signed-off-by: Jonathan Dieter <jdieter@gmail.com>
-rw-r--r--createprestorepo/ChangeLog2
-rw-r--r--presto-utils/COPYING (renamed from createprestorepo/COPYING)0
-rw-r--r--presto-utils/ChangeLog6
-rw-r--r--presto-utils/Makefile (renamed from createprestorepo/Makefile)2
-rw-r--r--presto-utils/README (renamed from createprestorepo/README)6
-rwxr-xr-xpresto-utils/createdeltarpms (renamed from createprestorepo/makedeltarepo)0
-rwxr-xr-xpresto-utils/createprestorepo (renamed from createprestorepo/createprestorepo)0
-rw-r--r--presto-utils/dumpMetadata.py (renamed from createprestorepo/dumpMetadata.py)1
-rwxr-xr-xpresto-utils/gendeltarpms.py (renamed from createprestorepo/gendeltarpms.py)170
-rwxr-xr-xpresto-utils/genprestometadata.py (renamed from createprestorepo/genprestometadata.py)104
-rw-r--r--presto-utils/packagelist.py (renamed from createprestorepo/packagelist.py)3
11 files changed, 229 insertions, 65 deletions
diff --git a/createprestorepo/ChangeLog b/createprestorepo/ChangeLog
deleted file mode 100644
index 2de29c0..0000000
--- a/createprestorepo/ChangeLog
+++ /dev/null
@@ -1,2 +0,0 @@
-* Tue May 7 2007 Jonathan Dieter <jdieter@gmail.com> - 0.1.0
- - Initial release
diff --git a/createprestorepo/COPYING b/presto-utils/COPYING
index e77696a..e77696a 100644
--- a/createprestorepo/COPYING
+++ b/presto-utils/COPYING
diff --git a/presto-utils/ChangeLog b/presto-utils/ChangeLog
new file mode 100644
index 0000000..1640457
--- /dev/null
+++ b/presto-utils/ChangeLog
@@ -0,0 +1,6 @@
+* Tue Jun 19 2007 Jonathan Dieter <jdieter@gmail.com> - 0.2.0
+ - Now works with createrepo and modifyrepo
+ - Many bugfixes
+
+* Tue May 7 2007 Jonathan Dieter <jdieter@gmail.com> - 0.1.0
+ - Initial release
diff --git a/createprestorepo/Makefile b/presto-utils/Makefile
index 51228f8..9e8c6eb 100644
--- a/createprestorepo/Makefile
+++ b/presto-utils/Makefile
@@ -3,7 +3,7 @@ clean:
install:
install -m 755 createprestorepo $(DESTDIR)/usr/bin/
- install -m 755 makedeltarepo $(DESTDIR)/usr/bin/
+ install -m 755 createdeltarpms $(DESTDIR)/usr/bin/
mkdir -p $(DESTDIR)/usr/share/createprestorepo
install -m 755 genprestometadata.py $(DESTDIR)/usr/share/createprestorepo
install -m 644 dumpMetadata.py $(DESTDIR)/usr/share/createprestorepo
diff --git a/createprestorepo/README b/presto-utils/README
index 010fdd3..a81187a 100644
--- a/createprestorepo/README
+++ b/presto-utils/README
@@ -10,9 +10,9 @@ Installation:
2- Run 'make install'
Running:
-1- First run 'makedeltarepo <repo dir> <drpm dir>' where <repo dir> is the base
- directory for your repository and <drpm dir> is the subdirectory you want to
- create the deltarpms into
+1- First run 'createdeltarpms <repo dir> <drpm dir>' where <repo dir> is the
+ base directory for your repository and <drpm dir> is the subdirectory you
+ want to create the deltarpms into
2- Run 'createprestorepo <repo dir>' where <repo dir> is the base directory for
your repository.
diff --git a/createprestorepo/makedeltarepo b/presto-utils/createdeltarpms
index 99bedcc..99bedcc 100755
--- a/createprestorepo/makedeltarepo
+++ b/presto-utils/createdeltarpms
diff --git a/createprestorepo/createprestorepo b/presto-utils/createprestorepo
index 5334a80..5334a80 100755
--- a/createprestorepo/createprestorepo
+++ b/presto-utils/createprestorepo
diff --git a/createprestorepo/dumpMetadata.py b/presto-utils/dumpMetadata.py
index 6969400..0ec2c20 100644
--- a/createprestorepo/dumpMetadata.py
+++ b/presto-utils/dumpMetadata.py
@@ -494,3 +494,4 @@ def repoXML(node, cmds):
timestamp = data.newChild(None, 'timestamp', str(timestamp))
unchecksum = data.newChild(None, 'open-checksum', uncsum)
unchecksum.newProp('type', sumtype)
+
diff --git a/createprestorepo/gendeltarpms.py b/presto-utils/gendeltarpms.py
index 26d795b..7db9d4f 100755
--- a/createprestorepo/gendeltarpms.py
+++ b/presto-utils/gendeltarpms.py
@@ -15,11 +15,12 @@
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
-import errno, os, sys, gzip
+import errno, os, sys
import fnmatch, re
import rpmUtils.transaction, rpmUtils.miscutils
-import commands, libxml2
+import commands
import string
+import getopt
from dumpMetadata import byteranges
from packagelist import RpmItem, DrpmItem, PackageList
@@ -27,9 +28,36 @@ import gzip
from zlib import error as zlibError
from gzip import write32u, FNAME
-
-DRPMWORTHKEEPINGTHRESH=0.7
+DRPMWORTHKEEPINGTHRESH=0.5
DEBUG=0
+__version__ = "0.2.0"
+
+def errorprint(stuff):
+ print >> sys.stderr, stuff
+
+def _(args):
+ """Stub function for translation"""
+ return args
+
+def usage(retval=1):
+ print _("""
+ createdeltarpms [options] directory-of-packages directory-of-deltarpms
+
+ Options:
+ -d, --dist-update <dir> = optional directory containing old distribution
+ to create an update from
+ -c, --count <number> = optional maximum number of deltarpms to create
+ per package. Default is maximum available
+ -n, --no-first = Don't create deltarpm against first rpm if number exceeds
+ count. Useful if you are running a continually-updated
+ distribution rather than one with set release cycles.
+ -q, --quiet = run quietly
+ -v, --verbose = run verbosely
+ -h, --help = show this help
+ -V, --version = output version
+ """)
+
+ sys.exit(retval)
def getDeltaNevr(filename):
"""Get nevr for src rpm of a deltarpm. Return a tuple of (n, e, v, r)"""
@@ -91,11 +119,7 @@ def getFileList(path, ext, filelist):
extlen = len(ext)
totalpath = os.path.normpath(path)
- try:
- dir_list = os.listdir(totalpath)
- except OSError, e:
- errorprint(_('Error accessing directory %s, %s') % (totalpath, e))
- sys.exit(1)
+ dir_list = os.listdir(totalpath)
for d in dir_list:
if os.path.isdir(totalpath + '/' + d):
@@ -105,7 +129,7 @@ def getFileList(path, ext, filelist):
return filelist
-def createPrestoRepo(base_dir, drpm_dir, dst_dir=None):
+def createPrestoRepo(base_dir, drpm_dir, dst_dir=None, DrpmsPerPackage=0, DoFirst=True):
ts = rpmUtils.transaction.initReadOnlyTransaction()
changed = False
@@ -114,11 +138,21 @@ def createPrestoRepo(base_dir, drpm_dir, dst_dir=None):
print 'Using base dir: %s' % base_dir
print 'Using destination dir: %s' % drpm_dir
- rpm_files = getFileList(base_dir, ".rpm", [])
+ try:
+ rpm_files = getFileList(base_dir, ".rpm", [])
+ except OSError, e:
+ errorprint(_("Error: Unable to find directory %s: %s" % (base_dir, e)))
+ return False
if dst_dir != None:
- dst_rpm_files = getFileList(dst_dir, ".rpm", [])
- drpm_files = getFileList(drpm_dir, ".drpm", [])
-
+ try:
+ dst_rpm_files = getFileList(dst_dir, ".rpm", [])
+ except OSError, e:
+ errorprint(_("Error: Unable to find directory %s: %s" % (dst_dir, e)))
+ return False
+ try:
+ drpm_files = getFileList(drpm_dir, ".drpm", [])
+ except:
+ drpm_files = []
if not len(rpm_files):
print ' Nothing found.'
return changed
@@ -192,25 +226,99 @@ def createPrestoRepo(base_dir, drpm_dir, dst_dir=None):
rpm_list.dump()
# Build deltarpms
- if dst_dir != None:
- rpm_list.makeDeltaRpms(DrpmsPerPackage=1, DoFirst=False)
+ rpm_list.makeDeltaRpms(DrpmsPerPackage, DoFirst)
+ return True
+
+def parseArgs(args):
+ """
+ Parse the command line args return a commands dict and directory.
+ Sanity check all the things being passed in.
+ """
+ cmds = {}
+ cmds['quiet'] = 0
+ cmds['verbose'] = 0
+ cmds['dist-update'] = None
+ cmds['count'] = None
+ cmds['do-first'] = 1
+ try:
+ gopts, argsleft = getopt.getopt(args, 'hqvVd:c:n', ['help', 'quiet', 'verbose', 'version', 'dist-update=', 'count=', 'no-first'])
+ except getopt.error, e:
+ errorprint(_('Options Error: %s.') % e)
+ usage()
+
+ try:
+ for arg,a in gopts:
+ if arg in ['-h','--help']:
+ usage(retval=0)
+ elif arg in ['-V', '--version']:
+ print '%s' % __version__
+ sys.exit(0)
+ except ValueError, e:
+ errorprint(_('Options Error: %s') % e)
+ usage()
+
+
+ # make sure our dir makes sense before we continue
+ if len(argsleft) > 2:
+ errorprint(_('Error: You may only specify one rpm directory and one deltarpm directory.'))
+ usage()
+ elif len(argsleft) == 0:
+ errorprint(_('Error: Must specify an rpm directory to index and a destination deltarpm directory.'))
+ usage()
else:
- rpm_list.makeDeltaRpms(DrpmsPerPackage=5)
+ directories = argsleft
+
+ try:
+ for arg,a in gopts:
+ if arg in ['-v', '--verbose']:
+ cmds['verbose'] = 1
+ elif arg in ["-q", '--quiet']:
+ cmds['quiet'] = 1
+ elif arg in ["-n", '--no-first']:
+ cmds['do-first'] = 0
+ elif arg in ['-d', '--dist-update']:
+ if cmds['dist-update'] is not None:
+ errorprint(_('Error: Only one directory for distribution updates allowed.'))
+ usage()
+ else:
+ cmds['dist-update'] = a
+ elif arg in ['-c', '--count']:
+ if cmds['count'] is not None:
+ errorprint(_('Error: Only one count allowed.'))
+ usage()
+ else:
+ cmds['count'] = int(a)
+
+ except ValueError, e:
+ errorprint(_('Options Error: %s') % e)
+ usage()
+
+ rpm_directory = directories[0]
+ drpm_directory = directories[1]
+
+ rpm_directory = os.path.normpath(rpm_directory)
+ drpm_directory = os.path.normpath(drpm_directory)
+
+ # Fixup first directory
+ directories[0] = rpm_directory
+ directories[1] = drpm_directory
+
+ # Set count
+ if cmds['count'] == None:
+ cmds['count'] = 0
+
+ return cmds, directories
if __name__ == '__main__':
- if len(sys.argv) < 2 or (sys.argv[1] == "-d" and len(sys.argv) < 4):
- print 'Usage: %s <bin_rpm_dir> <delta_rpm_dir>' % os.path.basename(sys.argv[0])
- print ' %s -d <old_dist_dir> <new_dist_dir> <delta_rpm_dir>' % os.path.basename(sys.argv[0])
- sys.exit(errno.EINVAL)
- if sys.argv[1] != "-d":
- bin_rpm_path = sys.argv[1]
- delta_rpm_path = sys.argv[2]
- createPrestoRepo(bin_rpm_path, delta_rpm_path)
+ cmds, directories = parseArgs(sys.argv[1:])
+ if cmds['dist-update'] != None:
+ if createPrestoRepo(cmds['dist-update'], directories[1], directories[0], cmds['count'], cmds['do-first']):
+ sys.exit(0)
+ else:
+ sys.exit(1)
else:
- src_bin_rpm_path = sys.argv[2]
- dst_bin_rpm_path = sys.argv[3]
- delta_rpm_path = sys.argv[4]
- createPrestoRepo(src_bin_rpm_path, delta_rpm_path, dst_bin_rpm_path)
-
- sys.exit(0)
+ if createPrestoRepo(directories[0], directories[1], None, cmds['count'], cmds['do-first']):
+ sys.exit(0)
+ else:
+ sys.exit(1)
diff --git a/createprestorepo/genprestometadata.py b/presto-utils/genprestometadata.py
index bbce09d..6788ffc 100755
--- a/createprestorepo/genprestometadata.py
+++ b/presto-utils/genprestometadata.py
@@ -27,10 +27,12 @@ import libxml2
import string
import fnmatch
import urlgrabber
+import commands
+import shutil
import dumpMetadata
from dumpMetadata import _gzipOpen
-__version__ = '0.4.3'
+__version__ = '0.2.0'
def errorprint(stuff):
print >> sys.stderr, stuff
@@ -47,6 +49,7 @@ def usage(retval=1):
-u, --baseurl <url> = optional base url location for all files
-o, --outputdir <dir> = optional directory to output to
-x, --exclude = files globs to exclude, can be specified multiple times
+ -m, --merge = merge with repository created by createrepo
-q, --quiet = run quietly
-v, --verbose = run verbosely
-c, --cachedir <dir> = specify which dir to use for the checksum cache
@@ -128,7 +131,10 @@ class MetaDataGenerator:
self.formatns = self.prestoroot.newNs('http://linux.duke.edu/metadata/rpm', 'rpm')
self.prestoroot.setNs(basens)
prestofilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestofile'])
- self.prestofile = _gzipOpen(prestofilepath, 'w')
+ if self.cmds['merge'] == 1:
+ self.prestofile = open(prestofilepath, 'w')
+ else:
+ self.prestofile = _gzipOpen(prestofilepath, 'w')
self.prestofile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
self.prestofile.write('<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">\n' %
self.pkgcount)
@@ -173,26 +179,37 @@ class MetaDataGenerator:
self.prestodoc.freeDoc()
def doRepoMetadata(self):
- """wrapper to generate the prestomd.xml file that stores the info on the other files"""
- repodoc = libxml2.newDoc("1.0")
- reporoot = repodoc.newChild(None, "repomd", None)
- repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None)
- reporoot.setNs(repons)
- repofilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestomdfile'])
+ if self.cmds['prestomdfile'] == None: # Merge
+ shutil.copyfile(os.path.join(self.cmds['outputdir'], self.cmds['finaldir'], "repomd.xml"), os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], "repomd.xml"))
+ command = "/usr/bin/modifyrepo %s %s" % (os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestofile']), os.path.join(self.cmds['outputdir'], self.cmds['tempdir']))
+ (code, out) = commands.getstatusoutput(command)
+ if code:
+ errorprint(_('\nAn error occurred trying to run modifyrepo: %s') % e)
+ sys.exit(1)
+ os.unlink(os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestofile']))
+ self.cmds['prestofile'] = "%s.gz" % self.cmds['prestofile']
+ self.cmds['prestomdfile'] = "repomd.xml"
+ else:
+ """wrapper to generate the prestomd.xml file that stores the info on the other files"""
+ repodoc = libxml2.newDoc("1.0")
+ reporoot = repodoc.newChild(None, "repomd", None)
+ repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None)
+ reporoot.setNs(repons)
+ repofilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestomdfile'])
- try:
- dumpMetadata.repoXML(reporoot, self.cmds)
- except dumpMetadata.MDError, e:
- errorprint(_('Error generating repo xml file: %s') % e)
- sys.exit(1)
+ try:
+ dumpMetadata.repoXML(reporoot, self.cmds)
+ except dumpMetadata.MDError, e:
+ errorprint(_('Error generating repo xml file: %s') % e)
+ sys.exit(1)
- try:
- repodoc.saveFormatFileEnc(repofilepath, 'UTF-8', 1)
- except:
- errorprint(_('Error saving temp file for rep xml: %s') % repofilepath)
- sys.exit(1)
+ try:
+ repodoc.saveFormatFileEnc(repofilepath, 'UTF-8', 1)
+ except:
+ errorprint(_('Error saving temp file for rep xml: %s') % repofilepath)
+ sys.exit(1)
- del repodoc
+ del repodoc
class SplitMetaDataGenerator(MetaDataGenerator):
@@ -270,6 +287,7 @@ def parseArgs(args):
cmds['excludes'] = []
cmds['baseurl'] = None
cmds['sumtype'] = 'sha'
+ cmds['merge'] = 0
cmds['pretty'] = 0
cmds['cachedir'] = None
cmds['basedir'] = os.getcwd()
@@ -280,7 +298,7 @@ def parseArgs(args):
cmds['dir-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*']
try:
- gopts, argsleft = getopt.getopt(args, 'phqVvs:x:u:c:o:', ['help', 'exclude=',
+ gopts, argsleft = getopt.getopt(args, 'phqVvms:x:u:c:o:', ['help', 'exclude=',
'quiet', 'verbose', 'cachedir=', 'basedir=',
'baseurl=', 'checksum=',
'version', 'pretty', 'split', 'outputdir='])
@@ -328,6 +346,8 @@ def parseArgs(args):
cmds['excludes'].append(a)
elif arg in ['-p', '--pretty']:
cmds['pretty'] = 1
+ elif arg in ['-m', '--merge']:
+ cmds['merge'] = 1
elif arg in ['-c', '--cachedir']:
cmds['cache'] = True
cmds['cachedir'] = a
@@ -363,8 +383,12 @@ def parseArgs(args):
cmds['cachedir'] = a
#setup some defaults
- cmds['prestofile'] = 'presto.xml.gz'
- cmds['prestomdfile'] = 'prestomd.xml'
+ if cmds['merge'] == 1:
+ cmds['prestofile'] = 'deltas.xml'
+ cmds['prestomdfile'] = None
+ else:
+ cmds['prestofile'] = 'presto.xml.gz'
+ cmds['prestomdfile'] = 'prestomd.xml'
cmds['tempdir'] = '.repodata'
cmds['finaldir'] = 'repodata'
cmds['olddir'] = '.olddata'
@@ -405,11 +429,12 @@ def main(args):
# make sure we can write to where we want to write to:
for direc in ['tempdir', 'finaldir']:
for file in ['prestofile', 'prestomdfile']:
- filepath = os.path.join(cmds['outputdir'], cmds[direc], cmds[file])
- if os.path.exists(filepath):
- if not os.access(filepath, os.W_OK):
- errorprint(_('error in must be able to write to metadata files:\n -> %s') % filepath)
- usage()
+ if cmds[file] != None:
+ filepath = os.path.join(cmds['outputdir'], cmds[direc], cmds[file])
+ if os.path.exists(filepath):
+ if not os.access(filepath, os.W_OK):
+ errorprint(_('error in must be able to write to metadata files:\n -> %s') % filepath)
+ usage()
if cmds['split']:
cmds['basedir'] = oldbase
@@ -454,6 +479,31 @@ def main(args):
sys.exit(1)
+ # Move everything else back from olddir (eg. repoview files)
+ olddir = os.path.join(cmds['outputdir'], cmds['olddir'])
+ finaldir = os.path.join(cmds['outputdir'], cmds['finaldir'])
+ for file in os.listdir(olddir):
+ oldfile = os.path.join(olddir, file)
+ finalfile = os.path.join(finaldir, file)
+ if os.path.exists(finalfile):
+ # Hmph? Just leave it alone, then.
+ try:
+ if os.path.isdir(oldfile):
+ shutil.rmtree(oldfile)
+ else:
+ os.remove(oldfile)
+ except OSError, e:
+ errorprint(_('Could not remove old non-metadata file: %s') % oldfile)
+ errorprint(_('Error was %s') % e)
+ sys.exit(1)
+ else:
+ try:
+ os.rename(oldfile, finalfile)
+ except OSError, e:
+ errorprint(_('Could not restore old non-metadata file: %s -> %s') % (oldfile, finalfile))
+ errorprint(_('Error was %s') % e)
+ sys.exit(1)
+
#XXX: fix to remove tree as we mung basedir
try:
os.rmdir(os.path.join(cmds['outputdir'], cmds['olddir']))
diff --git a/createprestorepo/packagelist.py b/presto-utils/packagelist.py
index c057f3f..e9019f9 100644
--- a/createprestorepo/packagelist.py
+++ b/presto-utils/packagelist.py
@@ -272,7 +272,7 @@ class PackageList:
return False
- def makeDeltaRpms(self, DrpmsPerPackage=0, OnlyLastPackage=True, DoFirst=True):
+ def makeDeltaRpms(self, DrpmsPerPackage=0, DoFirst=True, OnlyLastPackage=True):
"""Create deltarpms for packages.
Attributes:
OnlyLastPackage (Boolean): Only create deltarpms to point to last destination package
@@ -383,3 +383,4 @@ class PackageList:
if drpmsize > self.threshold * newsize:
return False
return True
+