From b1147b441a7f4873f39c5ce337d516b2cd1483eb Mon Sep 17 00:00:00 2001 From: Jonathan Dieter Date: Thu, 29 Mar 2007 17:39:26 +0300 Subject: New server Signed-off-by: Jonathan Dieter --- Makefile | 1 + makerepo/dumpMetadata.py | 346 +---------------------------- makerepo/genprestometadata.py | 496 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 507 insertions(+), 336 deletions(-) create mode 100644 makerepo/genprestometadata.py diff --git a/Makefile b/Makefile index a335109..5e1b97b 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ clean: rm -f *.pyc *.pyo *~ + cd makerepo; rm -f *.pyc *.pyo *~ cd shared; rm -f *.pyc *.pyo *~ install: diff --git a/makerepo/dumpMetadata.py b/makerepo/dumpMetadata.py index 4ca0793..2f074da 100755 --- a/makerepo/dumpMetadata.py +++ b/makerepo/dumpMetadata.py @@ -209,7 +209,7 @@ class MDError(exceptions.Exception): class RpmMetaData: - """each rpm is one object, you pass it an rpm file + """each drpm is one object, you pass it an rpm file it opens the file, and pulls the information out in bite-sized chunks :) """ @@ -236,65 +236,13 @@ class RpmMetaData: fo.close() del fo del fd - - # setup our regex objects - fileglobs = options['file-pattern-match'] - #['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$'] - dirglobs = options['dir-pattern-match'] - #['.*bin\/.*', '^\/etc\/.*'] - self.dirrc = [] - self.filerc = [] - for glob in fileglobs: - self.filerc.append(re.compile(glob)) - - for glob in dirglobs: - self.dirrc.append(re.compile(glob)) - - self.filenames = [] - self.dirnames = [] - self.ghostnames = [] - self.genFileLists() - + def arch(self): if self.tagByName('sourcepackage') == 1: return 'src' else: return self.tagByName('arch') - def _correctFlags(self, flags): - returnflags=[] - if flags is None: - return returnflags - - if type(flags) is not types.ListType: - newflag = flags & 0xf - returnflags.append(newflag) - else: - for flag in flags: - newflag = flag - if flag is not None: - newflag = flag & 0xf - returnflags.append(newflag) - return returnflags - - def _checkPreReq(self, flags): - reqs=[] - if flags is None: - return reqs - - if type(flags) is not types.ListType: - flags = [flags] - for flag in flags: - newflag = flag - if flag is not None: - newflag = flag & 64 - if newflag == 64: - reqs.append(1) - else: - reqs.append(0) - return reqs - - def _correctVersion(self, vers): returnvers = [] vertuple = (None, None, None) @@ -435,126 +383,12 @@ class RpmMetaData: lst.append(data) return lst - def epoch(self): if self.hdr['epoch'] is None: return 0 else: return self.tagByName('epoch') - - def genFileLists(self): - """produces lists of dirs and files for this header in two lists""" - - files = self.listTagByName('filenames') - fileflags = self.listTagByName('fileflags') - filemodes = self.listTagByName('filemodes') - filetuple = zip(files, filemodes, fileflags) - for (file, mode, flag) in filetuple: - #garbage checks - if mode is None or mode == '': - self.filenames.append(file) - continue - if not RpmMetaData.mode_cache.has_key(mode): - RpmMetaData.mode_cache[mode] = stat.S_ISDIR(mode) - if RpmMetaData.mode_cache[mode]: - self.dirnames.append(file) - else: - if flag is None: - self.filenames.append(file) - else: - if (flag & 64): - self.ghostnames.append(file) - continue - self.filenames.append(file) - - - def usefulFiles(self): - """search for good files""" - returns = {} - for item in self.filenames: - if item is None: - continue - for glob in self.filerc: - if glob.match(item): - returns[item] = 1 - return returns - def usefulGhosts(self): - """search for useful ghost file names""" - returns = {} - for item in self.ghostnames: - if item is None: - continue - for glob in self.filerc: - if glob.match(item): - returns[item] = 1 - return returns - - - def usefulDirs(self): - """search for good dirs""" - returns = {} - for item in self.dirnames: - if item is None: - continue - for glob in self.dirrc: - if glob.match(item): - returns[item] = 1 - return returns.keys() - - - def depsList(self): - """returns a list of tuples of dependencies""" - # these should probably compress down duplicates too - lst = [] - names = self.hdr[rpm.RPMTAG_REQUIRENAME] - tmpflags = self.hdr[rpm.RPMTAG_REQUIREFLAGS] - flags = self._correctFlags(tmpflags) - prereq = self._checkPreReq(tmpflags) - ver = self._correctVersion(self.hdr[rpm.RPMTAG_REQUIREVERSION]) - if names is not None: - lst = zip(names, flags, ver, prereq) - return self._uniq(lst) - - def obsoletesList(self): - lst = [] - names = self.hdr[rpm.RPMTAG_OBSOLETENAME] - tmpflags = self.hdr[rpm.RPMTAG_OBSOLETEFLAGS] - flags = self._correctFlags(tmpflags) - ver = self._correctVersion(self.hdr[rpm.RPMTAG_OBSOLETEVERSION]) - if names is not None: - lst = zip(names, flags, ver) - return self._uniq(lst) - - def conflictsList(self): - lst = [] - names = self.hdr[rpm.RPMTAG_CONFLICTNAME] - tmpflags = self.hdr[rpm.RPMTAG_CONFLICTFLAGS] - flags = self._correctFlags(tmpflags) - ver = self._correctVersion(self.hdr[rpm.RPMTAG_CONFLICTVERSION]) - if names is not None: - lst = zip(names, flags, ver) - return self._uniq(lst) - - def providesList(self): - lst = [] - names = self.hdr[rpm.RPMTAG_PROVIDENAME] - tmpflags = self.hdr[rpm.RPMTAG_PROVIDEFLAGS] - flags = self._correctFlags(tmpflags) - ver = self._correctVersion(self.hdr[rpm.RPMTAG_PROVIDEVERSION]) - if names is not None: - lst = zip(names, flags, ver) - return self._uniq(lst) - - def changelogLists(self): - lst = [] - names = self.listTagByName('changelogname') - times = self.listTagByName('changelogtime') - texts = self.listTagByName('changelogtext') - if len(names) > 0: - lst = zip(names, times, texts) - return lst - def doChecksumCache(self, fo): """return a checksum for a package: - check if the checksum cache is enabled @@ -583,161 +417,24 @@ class RpmMetaData: -def generateXML(doc, node, formatns, rpmObj, sumtype): +def generateXML(doc, node, formatns, drpmObj, sumtype): """takes an xml doc object and a package metadata entry node, populates a package node with the md information""" ns = node.ns() pkgNode = node.newChild(None, "package", None) pkgNode.newProp('type', 'rpm') - pkgNode.newChild(None, 'name', rpmObj.tagByName('name')) - pkgNode.newChild(None, 'arch', rpmObj.arch()) + pkgNode.newChild(None, 'name', drpmObj.tagByName('name')) + pkgNode.newChild(None, 'arch', drpmObj.arch()) version = pkgNode.newChild(None, 'version', None) - version.newProp('epoch', str(rpmObj.epoch())) - version.newProp('ver', str(rpmObj.tagByName('version'))) - version.newProp('rel', str(rpmObj.tagByName('release'))) - csum = pkgNode.newChild(None, 'checksum', rpmObj.pkgid) - csum.newProp('type', sumtype) - csum.newProp('pkgid', 'YES') - for tag in ['summary', 'description', 'packager', 'url']: - value = rpmObj.tagByName(tag) - value = utf8String(value) - value = re.sub("\n$", '', value) - entry = pkgNode.newChild(None, tag, None) - entry.addContent(value) - - time = pkgNode.newChild(None, 'time', None) - time.newProp('file', str(rpmObj.mtime)) - time.newProp('build', str(rpmObj.tagByName('buildtime'))) - size = pkgNode.newChild(None, 'size', None) - size.newProp('package', str(rpmObj.size)) - size.newProp('installed', str(rpmObj.tagByName('size'))) - size.newProp('archive', str(rpmObj.tagByName('archivesize'))) - location = pkgNode.newChild(None, 'location', None) - if rpmObj.localurl is not None: - location.newProp('xml:base', rpmObj.localurl) - location.newProp('href', rpmObj.relativepath) - format = pkgNode.newChild(ns, 'format', None) - for tag in ['license', 'vendor', 'group', 'buildhost', 'sourcerpm']: - value = rpmObj.tagByName(tag) - value = utf8String(value) - value = re.sub("\n$", '', value) - entry = format.newChild(formatns, tag, None) - entry.addContent(value) - - hr = format.newChild(formatns, 'header-range', None) - hr.newProp('start', str(rpmObj.rangestart)) - hr.newProp('end', str(rpmObj.rangeend)) - for (lst, nodename) in [(rpmObj.providesList(), 'provides'), - (rpmObj.conflictsList(), 'conflicts'), - (rpmObj.obsoletesList(), 'obsoletes')]: - if len(lst) > 0: - rpconode = format.newChild(formatns, nodename, None) - for (name, flags, (e,v,r)) in lst: - entry = rpconode.newChild(formatns, 'entry', None) - entry.newProp('name', name) - if flags != 0: - if flags == 2: arg = 'LT' - if flags == 4: arg = 'GT' - if flags == 8: arg = 'EQ' - if flags == 10: arg = 'LE' - if flags == 12: arg = 'GE' - entry.newProp('flags', arg) - # if we've got a flag we've got a version, I hope :) - if e: - entry.newProp('epoch', str(e)) - if v: - entry.newProp('ver', str(v)) - if r: - entry.newProp('rel', str(r)) - - depsList = rpmObj.depsList() - if len(depsList) > 0: - rpconode = format.newChild(formatns, 'requires', None) - for (name, flags, (e,v,r), prereq) in depsList: - entry = rpconode.newChild(formatns, 'entry', None) - entry.newProp('name', name) - if flags != 0: - if flags == 2: arg = 'LT' - if flags == 4: arg = 'GT' - if flags == 8: arg = 'EQ' - if flags == 10: arg = 'LE' - if flags == 12: arg = 'GE' - entry.newProp('flags', arg) - # if we've got a flag we've got a version, I hope :) - if e: - entry.newProp('epoch', str(e)) - if v: - entry.newProp('ver', str(v)) - if r: - entry.newProp('rel', str(r)) - if prereq == 1: - entry.newProp('pre', str(prereq)) - - for file in rpmObj.usefulFiles(): - files = format.newChild(None, 'file', None) - file = utf8String(file) - files.addContent(file) - for directory in rpmObj.usefulDirs(): - files = format.newChild(None, 'file', None) - directory = utf8String(directory) - files.addContent(directory) - files.newProp('type', 'dir') - for directory in rpmObj.usefulGhosts(): - files = format.newChild(None, 'file', None) - directory = utf8String(directory) - files.addContent(directory) - files.newProp('type', 'ghost') - + version.newProp('epoch', str(drpmObj.epoch())) + version.newProp('ver', str(drpmObj.tagByName('version'))) + version.newProp('rel', str(drpmObj.tagByName('release'))) return pkgNode - -def fileListXML(doc, node, rpmObj): - pkg = node.newChild(None, 'package', None) - pkg.newProp('pkgid', rpmObj.pkgid) - pkg.newProp('name', rpmObj.tagByName('name')) - pkg.newProp('arch', rpmObj.arch()) - version = pkg.newChild(None, 'version', None) - version.newProp('epoch', str(rpmObj.epoch())) - version.newProp('ver', str(rpmObj.tagByName('version'))) - version.newProp('rel', str(rpmObj.tagByName('release'))) - for file in rpmObj.filenames: - files = pkg.newChild(None, 'file', None) - file = utf8String(file) - files.addContent(file) - for directory in rpmObj.dirnames: - files = pkg.newChild(None, 'file', None) - directory = utf8String(directory) - files.addContent(directory) - files.newProp('type', 'dir') - for ghost in rpmObj.ghostnames: - files = pkg.newChild(None, 'file', None) - ghost = utf8String(ghost) - files.addContent(ghost) - files.newProp('type', 'ghost') - return pkg - -def otherXML(doc, node, rpmObj): - pkg = node.newChild(None, 'package', None) - pkg.newProp('pkgid', rpmObj.pkgid) - pkg.newProp('name', rpmObj.tagByName('name')) - pkg.newProp('arch', rpmObj.arch()) - version = pkg.newChild(None, 'version', None) - version.newProp('epoch', str(rpmObj.epoch())) - version.newProp('ver', str(rpmObj.tagByName('version'))) - version.newProp('rel', str(rpmObj.tagByName('release'))) - clogs = rpmObj.changelogLists() - for (name, time, text) in clogs: - clog = pkg.newChild(None, 'changelog', None) - clog.addContent(utf8String(text)) - clog.newProp('author', utf8String(name)) - clog.newProp('date', str(time)) - return pkg - + def repoXML(node, cmds): """generate the repomd.xml file that stores the info on the other files""" sumtype = cmds['sumtype'] - workfiles = [(cmds['otherfile'], 'other',), - (cmds['filelistsfile'], 'filelists'), - (cmds['primaryfile'], 'primary')] + workfiles = [(cmds['prestofile'], 'deltas')] for (file, ftype) in workfiles: @@ -757,26 +454,3 @@ def repoXML(node, cmds): timestamp = data.newChild(None, 'timestamp', str(timestamp)) unchecksum = data.newChild(None, 'open-checksum', uncsum) unchecksum.newProp('type', sumtype) - - # if we've got a group file then checksum it once and be done - if cmds['groupfile'] is not None: - grpfile = cmds['groupfile'] - timestamp = os.stat(grpfile)[8] - sfile = os.path.basename(grpfile) - fo = open(grpfile, 'r') - output = open(os.path.join(cmds['outputdir'], cmds['tempdir'], sfile), 'w') - output.write(fo.read()) - output.close() - fo.seek(0) - csum = getChecksum(sumtype, fo) - fo.close() - - data = node.newChild(None, 'data', None) - data.newProp('type', 'group') - location = data.newChild(None, 'location', None) - if cmds['baseurl'] is not None: - location.newProp('xml:base', cmds['baseurl']) - location.newProp('href', os.path.join(cmds['finaldir'], sfile)) - checksum = data.newChild(None, 'checksum', csum) - checksum.newProp('type', sumtype) - timestamp = data.newChild(None, 'timestamp', str(timestamp)) diff --git a/makerepo/genprestometadata.py b/makerepo/genprestometadata.py new file mode 100644 index 0000000..402af25 --- /dev/null +++ b/makerepo/genprestometadata.py @@ -0,0 +1,496 @@ +#!/usr/bin/python -t +# primary functions and glue for generating the repository metadata +# + +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Library General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. +# Copyright 2004 Duke University +# Copyright 2007 Jonathan Dieter + + +import os +import sys +import getopt +import rpm +import libxml2 +import string +import fnmatch +import urlgrabber + +import dumpMetadata +from dumpMetadata import _gzipOpen +__version__ = '0.4.3' + +def errorprint(stuff): + print >> sys.stderr, stuff + +def _(args): + """Stub function for translation""" + return args + +def usage(retval=1): + print _(""" + createrepo [options] directory-of-packages + + Options: + -u, --baseurl = optional base url location for all files + -o, --outputdir = optional directory to output to + -x, --exclude = files globs to exclude, can be specified multiple times + -q, --quiet = run quietly + -v, --verbose = run verbosely + -c, --cachedir = specify which dir to use for the checksum cache + -h, --help = show this help + -V, --version = output version + -p, --pretty = output xml files in pretty format. + """) + + sys.exit(retval) + +class MetaDataGenerator: + def __init__(self, cmds): + self.cmds = cmds + self.ts = rpm.TransactionSet() + self.pkgcount = 0 + self.files = [] + + def getFileList(self, basepath, path, ext, filelist): + """Return all files in path matching ext, store them in filelist, + recurse dirs. Returns a list object""" + + extlen = len(ext) + totalpath = os.path.normpath(os.path.join(basepath, path)) + try: + dir_list = os.listdir(totalpath) + except OSError, e: + errorprint(_('Error accessing directory %s, %s') % (totalpath, e)) + sys.exit(1) + + for d in dir_list: + if os.path.isdir(totalpath + '/' + d): + filelist = self.getFileList(basepath, os.path.join(path, d), ext, filelist) + else: + if string.lower(d[-extlen:]) == '%s' % (ext): + if totalpath.find(basepath) == 0: + relativepath = totalpath.replace(basepath, "", 1) + relativepath = relativepath.lstrip("/") + filelist.append(os.path.join(relativepath, d)) + else: + raise "basepath '%s' not found in path '%s'" % (basepath, totalpath) + + return filelist + + + def trimRpms(self, files): + badrpms = [] + for file in files: + for glob in self.cmds['excludes']: + if fnmatch.fnmatch(file, glob): + # print 'excluded: %s' % file + if file not in badrpms: + badrpms.append(file) + for file in badrpms: + if file in files: + files.remove(file) + return files + + def doPkgMetadata(self, directory): + """all the heavy lifting for the package metadata""" + + # rpms we're going to be dealing with + files = self.getFileList(self.cmds['basedir'], directory, '.dpm', []) + files = self.trimRpms(files) + self.pkgcount = len(files) + self.openMetadataDocs() + self.writeMetadataDocs(files) + self.closeMetadataDocs() + + + def openMetadataDocs(self): + self._setupPresto() + + def _setupPresto(self): + # setup the base metadata doc + self.prestodoc = libxml2.newDoc("1.0") + self.prestoroot = self.prestodoc.newChild(None, "metadata", None) + basens = self.prestoroot.newNs('http://linux.duke.edu/metadata/common', None) + self.formatns = self.prestoroot.newNs('http://linux.duke.edu/metadata/rpm', 'rpm') + self.prestoroot.setNs(basens) + prestofilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestofile']) + self.prestofile = _gzipOpen(prestofilepath, 'w') + self.prestofile.write('\n') + self.prestofile.write('\n' % + self.pkgcount) + + + def writeMetadataDocs(self, files, current=0): + for file in files: + current+=1 + try: + mdobj = dumpMetadata.RpmMetaData(self.ts, self.cmds['basedir'], file, self.cmds) + if not self.cmds['quiet']: + if self.cmds['verbose']: + print '%d/%d - %s' % (current, len(files), file) + else: + sys.stdout.write('\r' + ' ' * 80) + sys.stdout.write("\r%d/%d - %s" % (current, self.pkgcount, file)) + sys.stdout.flush() + except dumpMetadata.MDError, e: + errorprint('\n%s - %s' % (e, file)) + continue + else: + try: + node = dumpMetadata.generateXML(self.prestodoc, self.prestoroot, self.formatns, mdobj, self.cmds['sumtype']) + except dumpMetadata.MDError, e: + errorprint(_('\nAn error occurred creating presto metadata: %s') % e) + continue + else: + output = node.serialize('UTF-8', self.cmds['pretty']) + self.prestofile.write(output) + self.prestofile.write('\n') + node.unlinkNode() + node.freeNode() + del node + + return current + + + def closeMetadataDocs(self): + if not self.cmds['quiet']: + print '' + + # save them up to the tmp locations: + if not self.cmds['quiet']: + print _('Saving Presto metadata') + self.prestofile.write('\n') + self.prestofile.close() + self.prestodoc.freeDoc() + + def doRepoMetadata(self): + """wrapper to generate the prestomd.xml file that stores the info on the other files""" + repodoc = libxml2.newDoc("1.0") + reporoot = repodoc.newChild(None, "repomd", None) + repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None) + reporoot.setNs(repons) + repofilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestomdfile']) + + try: + dumpMetadata.repoXML(reporoot, self.cmds) + except dumpMetadata.MDError, e: + errorprint(_('Error generating repo xml file: %s') % e) + sys.exit(1) + + try: + repodoc.saveFormatFileEnc(repofilepath, 'UTF-8', 1) + except: + errorprint(_('Error saving temp file for rep xml: %s') % repofilepath) + sys.exit(1) + + del repodoc + +class SplitMetaDataGenerator(MetaDataGenerator): + + def __init__(self, cmds): + MetaDataGenerator.__init__(self, cmds) + self.initialdir = self.cmds['basedir'] + + def _getFragmentUrl(self, url, fragment): + import urlparse + urlparse.uses_fragment.append('media') + if not url: + return url + (scheme, netloc, path, query, fragid) = urlparse.urlsplit(url) + return urlparse.urlunsplit((scheme, netloc, path, query, str(fragment))) + + def doPkgMetadata(self, directories): + """all the heavy lifting for the package metadata""" + import types + if type(directories) == types.StringType: + MetaDataGenerator.doPkgMetadata(self, directories) + return + filematrix = {} + for mydir in directories: + filematrix[mydir] = self.getFileList(os.path.join(self.initialdir, mydir), '.', '.rpm', []) + self.trimRpms(filematrix[mydir]) + self.pkgcount += len(filematrix[mydir]) + + mediano = 1 + current = 0 + self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], mediano) + self.cmds['basedir'] = os.path.join(self.initialdir, directories[0]) + self.openMetadataDocs() + for mydir in directories: + self.cmds['basedir'] = os.path.join(self.initialdir, mydir) + self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], mediano) + current = self.writeMetadataDocs(filematrix[mydir], current) + mediano += 1 + self.cmds['basedir'] = os.path.join(self.initialdir, directories[0]) + self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], 1) + self.closeMetadataDocs() + + +def checkAndMakeDir(dir): + """ + check out the dir and make it, if possible, return 1 if done, else return 0 + """ + if os.path.exists(dir): + if not os.path.isdir(dir): + errorprint(_('%s is not a dir') % dir) + result = False + else: + if not os.access(dir, os.W_OK): + errorprint(_('%s is not writable') % dir) + result = False + else: + result = True + else: + try: + os.mkdir(dir) + except OSError, e: + errorprint(_('Error creating dir %s: %s') % (dir, e)) + result = False + else: + result = True + return result + +def parseArgs(args): + """ + Parse the command line args return a commands dict and directory. + Sanity check all the things being passed in. + """ + cmds = {} + cmds['quiet'] = 0 + cmds['verbose'] = 0 + cmds['excludes'] = [] + cmds['baseurl'] = None + cmds['sumtype'] = 'sha' + cmds['pretty'] = 0 + cmds['cachedir'] = None + cmds['basedir'] = os.getcwd() + cmds['cache'] = False + cmds['split'] = False + cmds['outputdir'] = "" + cmds['file-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$'] + cmds['dir-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*'] + + try: + gopts, argsleft = getopt.getopt(args, 'phqVvs:x:u:c:o:', ['help', 'exclude=', + 'quiet', 'verbose', 'cachedir=', 'basedir=', + 'baseurl=', 'checksum=', + 'version', 'pretty', 'split', 'outputdir=']) + except getopt.error, e: + errorprint(_('Options Error: %s.') % e) + usage() + + try: + for arg,a in gopts: + if arg in ['-h','--help']: + usage(retval=0) + elif arg in ['-V', '--version']: + print '%s' % __version__ + sys.exit(0) + elif arg == '--split': + cmds['split'] = True + except ValueError, e: + errorprint(_('Options Error: %s') % e) + usage() + + + # make sure our dir makes sense before we continue + if len(argsleft) > 1 and not cmds['split']: + errorprint(_('Error: Only one directory allowed per run.')) + usage() + elif len(argsleft) == 0: + errorprint(_('Error: Must specify a directory to index.')) + usage() + else: + directories = argsleft + + try: + for arg,a in gopts: + if arg in ['-v', '--verbose']: + cmds['verbose'] = 1 + elif arg in ["-q", '--quiet']: + cmds['quiet'] = 1 + elif arg in ['-u', '--baseurl']: + if cmds['baseurl'] is not None: + errorprint(_('Error: Only one baseurl allowed.')) + usage() + else: + cmds['baseurl'] = a + elif arg in ['-x', '--exclude']: + cmds['excludes'].append(a) + elif arg in ['-p', '--pretty']: + cmds['pretty'] = 1 + elif arg in ['-c', '--cachedir']: + cmds['cache'] = True + cmds['cachedir'] = a + elif arg == '--basedir': + cmds['basedir'] = a + elif arg in ['-o','--outputdir']: + cmds['outputdir'] = a + + except ValueError, e: + errorprint(_('Options Error: %s') % e) + usage() + + directory = directories[0] +# Fix paths + directory = os.path.normpath(directory) + if cmds['split']: + pass + elif os.path.isabs(directory): + cmds['basedir'] = directory + directory = '.' + else: + cmds['basedir'] = os.path.realpath(os.path.join(cmds['basedir'], directory)) + directory = '.' + if not cmds['outputdir']: + cmds['outputdir'] = cmds['basedir'] + if cmds['groupfile']: + a = cmds['groupfile'] + if cmds['split']: + a = os.path.join(cmds['basedir'], directory, cmds['groupfile']) + elif not os.path.isabs(a): + a = os.path.join(cmds['basedir'], cmds['groupfile']) + if not os.path.exists(a): + errorprint(_('Error: groupfile %s cannot be found.' % a)) + usage() + cmds['groupfile'] = a + if cmds['cachedir']: + a = cmds ['cachedir'] + if not os.path.isabs(a): + a = os.path.join(cmds['basedir'] ,a) + if not checkAndMakeDir(a): + errorprint(_('Error: cannot open/write to cache dir %s' % a)) + usage() + cmds['cachedir'] = a + + #setup some defaults + cmds['prestofile'] = 'presto.xml.gz' + cmds['prestomdfile'] = 'prestomd.xml' + cmds['tempdir'] = '.repodata' + cmds['finaldir'] = 'repodata' + cmds['olddir'] = '.olddata' + + # Fixup first directory + directories[0] = directory + return cmds, directories + +def main(args): + cmds, directories = parseArgs(args) + directory = directories[0] + # start the sanity/stupidity checks + if not os.path.exists(os.path.join(cmds['basedir'], directory)): + errorprint(_('Directory must exist')) + sys.exit(1) + + if not os.path.isdir(os.path.join(cmds['basedir'], directory)): + errorprint(_('Directory of packages must be a directory.')) + sys.exit(1) + + if not os.access(cmds['outputdir'], os.W_OK): + errorprint(_('Directory must be writable.')) + sys.exit(1) + + if cmds['split']: + oldbase = cmds['basedir'] + cmds['basedir'] = os.path.join(cmds['basedir'], directory) + if not checkAndMakeDir(os.path.join(cmds['outputdir'], cmds['tempdir'])): + sys.exit(1) + + if not checkAndMakeDir(os.path.join(cmds['outputdir'], cmds['finaldir'])): + sys.exit(1) + + if os.path.exists(os.path.join(cmds['outputdir'], cmds['olddir'])): + errorprint(_('Old data directory exists, please remove: %s') % cmds['olddir']) + sys.exit(1) + + # make sure we can write to where we want to write to: + for direc in ['tempdir', 'finaldir']: + for file in ['prestofile', 'prestomdfile']: + filepath = os.path.join(cmds['outputdir'], cmds[direc], cmds[file]) + if os.path.exists(filepath): + if not os.access(filepath, os.W_OK): + errorprint(_('error in must be able to write to metadata files:\n -> %s') % filepath) + usage() + + if cmds['split']: + cmds['basedir'] = oldbase + mdgen = SplitMetaDataGenerator(cmds) + mdgen.doPkgMetadata(directories) + else: + mdgen = MetaDataGenerator(cmds) + mdgen.doPkgMetadata(directory) + mdgen.doRepoMetadata() + + if os.path.exists(os.path.join(cmds['outputdir'], cmds['finaldir'])): + try: + os.rename(os.path.join(cmds['outputdir'], cmds['finaldir']), + os.path.join(cmds['outputdir'], cmds['olddir'])) + except: + errorprint(_('Error moving final %s to old dir %s' % (os.path.join(cmds['outputdir'], cmds['finaldir']), + os.path.join(cmds['outputdir'], cmds['olddir'])))) + sys.exit(1) + + try: + os.rename(os.path.join(cmds['outputdir'], cmds['tempdir']), + os.path.join(cmds['outputdir'], cmds['finaldir'])) + except: + errorprint(_('Error moving final metadata into place')) + # put the old stuff back + os.rename(os.path.join(cmds['outputdir'], cmds['olddir']), + os.path.join(cmds['outputdir'], cmds['finaldir'])) + sys.exit(1) + + for file in ['prestofile', 'prestomdfile']: + if cmds[file]: + fn = os.path.basename(cmds[file]) + else: + continue + oldfile = os.path.join(cmds['outputdir'], cmds['olddir'], fn) + if os.path.exists(oldfile): + try: + os.remove(oldfile) + except OSError, e: + errorprint(_('Could not remove old metadata file: %s') % oldfile) + errorprint(_('Error was %s') % e) + sys.exit(1) + + # Clean up any update metadata + mdpath = os.path.join(cmds['basedir'], cmds['olddir'], cmds['update-info-dir']) + if os.path.isdir(mdpath): + for file in os.listdir(mdpath): + os.remove(os.path.join(mdpath, file)) + os.rmdir(mdpath) + + +#XXX: fix to remove tree as we mung basedir + try: + os.rmdir(os.path.join(cmds['outputdir'], cmds['olddir'])) + except OSError, e: + errorprint(_('Could not remove old metadata dir: %s') % cmds['olddir']) + errorprint(_('Error was %s') % e) + errorprint(_('Please clean up this directory manually.')) + +if __name__ == "__main__": + if len(sys.argv) > 1: + if sys.argv[1] == 'profile': + import hotshot + p = hotshot.Profile(os.path.expanduser("~/createprestorepo.prof")) + p.run('main(sys.argv[2:])') + p.close() + else: + main(sys.argv[1:]) + else: + main(sys.argv[1:]) -- cgit