summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJonathan Dieter <jdieter@gmail.com>2007-03-29 17:39:26 +0300
committerJonathan Dieter <jdieter@gmail.com>2007-03-29 17:39:26 +0300
commitb1147b441a7f4873f39c5ce337d516b2cd1483eb (patch)
treeefcf451769cf54c8948bad77803baf90f7ccb068
parent3304f77f5f30e94c13147aa73eecf3d8cf9ef9aa (diff)
downloadpresto-b1147b441a7f4873f39c5ce337d516b2cd1483eb.tar.gz
presto-b1147b441a7f4873f39c5ce337d516b2cd1483eb.tar.xz
presto-b1147b441a7f4873f39c5ce337d516b2cd1483eb.zip
New server
Signed-off-by: Jonathan Dieter <jdieter@gmail.com>
-rw-r--r--Makefile1
-rwxr-xr-xmakerepo/dumpMetadata.py346
-rw-r--r--makerepo/genprestometadata.py496
3 files changed, 507 insertions, 336 deletions
diff --git a/Makefile b/Makefile
index a335109..5e1b97b 100644
--- a/Makefile
+++ b/Makefile
@@ -1,5 +1,6 @@
clean:
rm -f *.pyc *.pyo *~
+ cd makerepo; rm -f *.pyc *.pyo *~
cd shared; rm -f *.pyc *.pyo *~
install:
diff --git a/makerepo/dumpMetadata.py b/makerepo/dumpMetadata.py
index 4ca0793..2f074da 100755
--- a/makerepo/dumpMetadata.py
+++ b/makerepo/dumpMetadata.py
@@ -209,7 +209,7 @@ class MDError(exceptions.Exception):
class RpmMetaData:
- """each rpm is one object, you pass it an rpm file
+ """each drpm is one object, you pass it an rpm file
it opens the file, and pulls the information out in bite-sized chunks :)
"""
@@ -236,65 +236,13 @@ class RpmMetaData:
fo.close()
del fo
del fd
-
- # setup our regex objects
- fileglobs = options['file-pattern-match']
- #['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$']
- dirglobs = options['dir-pattern-match']
- #['.*bin\/.*', '^\/etc\/.*']
- self.dirrc = []
- self.filerc = []
- for glob in fileglobs:
- self.filerc.append(re.compile(glob))
-
- for glob in dirglobs:
- self.dirrc.append(re.compile(glob))
-
- self.filenames = []
- self.dirnames = []
- self.ghostnames = []
- self.genFileLists()
-
+
def arch(self):
if self.tagByName('sourcepackage') == 1:
return 'src'
else:
return self.tagByName('arch')
- def _correctFlags(self, flags):
- returnflags=[]
- if flags is None:
- return returnflags
-
- if type(flags) is not types.ListType:
- newflag = flags & 0xf
- returnflags.append(newflag)
- else:
- for flag in flags:
- newflag = flag
- if flag is not None:
- newflag = flag & 0xf
- returnflags.append(newflag)
- return returnflags
-
- def _checkPreReq(self, flags):
- reqs=[]
- if flags is None:
- return reqs
-
- if type(flags) is not types.ListType:
- flags = [flags]
- for flag in flags:
- newflag = flag
- if flag is not None:
- newflag = flag & 64
- if newflag == 64:
- reqs.append(1)
- else:
- reqs.append(0)
- return reqs
-
-
def _correctVersion(self, vers):
returnvers = []
vertuple = (None, None, None)
@@ -435,126 +383,12 @@ class RpmMetaData:
lst.append(data)
return lst
-
def epoch(self):
if self.hdr['epoch'] is None:
return 0
else:
return self.tagByName('epoch')
-
- def genFileLists(self):
- """produces lists of dirs and files for this header in two lists"""
-
- files = self.listTagByName('filenames')
- fileflags = self.listTagByName('fileflags')
- filemodes = self.listTagByName('filemodes')
- filetuple = zip(files, filemodes, fileflags)
- for (file, mode, flag) in filetuple:
- #garbage checks
- if mode is None or mode == '':
- self.filenames.append(file)
- continue
- if not RpmMetaData.mode_cache.has_key(mode):
- RpmMetaData.mode_cache[mode] = stat.S_ISDIR(mode)
- if RpmMetaData.mode_cache[mode]:
- self.dirnames.append(file)
- else:
- if flag is None:
- self.filenames.append(file)
- else:
- if (flag & 64):
- self.ghostnames.append(file)
- continue
- self.filenames.append(file)
-
-
- def usefulFiles(self):
- """search for good files"""
- returns = {}
- for item in self.filenames:
- if item is None:
- continue
- for glob in self.filerc:
- if glob.match(item):
- returns[item] = 1
- return returns
- def usefulGhosts(self):
- """search for useful ghost file names"""
- returns = {}
- for item in self.ghostnames:
- if item is None:
- continue
- for glob in self.filerc:
- if glob.match(item):
- returns[item] = 1
- return returns
-
-
- def usefulDirs(self):
- """search for good dirs"""
- returns = {}
- for item in self.dirnames:
- if item is None:
- continue
- for glob in self.dirrc:
- if glob.match(item):
- returns[item] = 1
- return returns.keys()
-
-
- def depsList(self):
- """returns a list of tuples of dependencies"""
- # these should probably compress down duplicates too
- lst = []
- names = self.hdr[rpm.RPMTAG_REQUIRENAME]
- tmpflags = self.hdr[rpm.RPMTAG_REQUIREFLAGS]
- flags = self._correctFlags(tmpflags)
- prereq = self._checkPreReq(tmpflags)
- ver = self._correctVersion(self.hdr[rpm.RPMTAG_REQUIREVERSION])
- if names is not None:
- lst = zip(names, flags, ver, prereq)
- return self._uniq(lst)
-
- def obsoletesList(self):
- lst = []
- names = self.hdr[rpm.RPMTAG_OBSOLETENAME]
- tmpflags = self.hdr[rpm.RPMTAG_OBSOLETEFLAGS]
- flags = self._correctFlags(tmpflags)
- ver = self._correctVersion(self.hdr[rpm.RPMTAG_OBSOLETEVERSION])
- if names is not None:
- lst = zip(names, flags, ver)
- return self._uniq(lst)
-
- def conflictsList(self):
- lst = []
- names = self.hdr[rpm.RPMTAG_CONFLICTNAME]
- tmpflags = self.hdr[rpm.RPMTAG_CONFLICTFLAGS]
- flags = self._correctFlags(tmpflags)
- ver = self._correctVersion(self.hdr[rpm.RPMTAG_CONFLICTVERSION])
- if names is not None:
- lst = zip(names, flags, ver)
- return self._uniq(lst)
-
- def providesList(self):
- lst = []
- names = self.hdr[rpm.RPMTAG_PROVIDENAME]
- tmpflags = self.hdr[rpm.RPMTAG_PROVIDEFLAGS]
- flags = self._correctFlags(tmpflags)
- ver = self._correctVersion(self.hdr[rpm.RPMTAG_PROVIDEVERSION])
- if names is not None:
- lst = zip(names, flags, ver)
- return self._uniq(lst)
-
- def changelogLists(self):
- lst = []
- names = self.listTagByName('changelogname')
- times = self.listTagByName('changelogtime')
- texts = self.listTagByName('changelogtext')
- if len(names) > 0:
- lst = zip(names, times, texts)
- return lst
-
def doChecksumCache(self, fo):
"""return a checksum for a package:
- check if the checksum cache is enabled
@@ -583,161 +417,24 @@ class RpmMetaData:
-def generateXML(doc, node, formatns, rpmObj, sumtype):
+def generateXML(doc, node, formatns, drpmObj, sumtype):
"""takes an xml doc object and a package metadata entry node, populates a
package node with the md information"""
ns = node.ns()
pkgNode = node.newChild(None, "package", None)
pkgNode.newProp('type', 'rpm')
- pkgNode.newChild(None, 'name', rpmObj.tagByName('name'))
- pkgNode.newChild(None, 'arch', rpmObj.arch())
+ pkgNode.newChild(None, 'name', drpmObj.tagByName('name'))
+ pkgNode.newChild(None, 'arch', drpmObj.arch())
version = pkgNode.newChild(None, 'version', None)
- version.newProp('epoch', str(rpmObj.epoch()))
- version.newProp('ver', str(rpmObj.tagByName('version')))
- version.newProp('rel', str(rpmObj.tagByName('release')))
- csum = pkgNode.newChild(None, 'checksum', rpmObj.pkgid)
- csum.newProp('type', sumtype)
- csum.newProp('pkgid', 'YES')
- for tag in ['summary', 'description', 'packager', 'url']:
- value = rpmObj.tagByName(tag)
- value = utf8String(value)
- value = re.sub("\n$", '', value)
- entry = pkgNode.newChild(None, tag, None)
- entry.addContent(value)
-
- time = pkgNode.newChild(None, 'time', None)
- time.newProp('file', str(rpmObj.mtime))
- time.newProp('build', str(rpmObj.tagByName('buildtime')))
- size = pkgNode.newChild(None, 'size', None)
- size.newProp('package', str(rpmObj.size))
- size.newProp('installed', str(rpmObj.tagByName('size')))
- size.newProp('archive', str(rpmObj.tagByName('archivesize')))
- location = pkgNode.newChild(None, 'location', None)
- if rpmObj.localurl is not None:
- location.newProp('xml:base', rpmObj.localurl)
- location.newProp('href', rpmObj.relativepath)
- format = pkgNode.newChild(ns, 'format', None)
- for tag in ['license', 'vendor', 'group', 'buildhost', 'sourcerpm']:
- value = rpmObj.tagByName(tag)
- value = utf8String(value)
- value = re.sub("\n$", '', value)
- entry = format.newChild(formatns, tag, None)
- entry.addContent(value)
-
- hr = format.newChild(formatns, 'header-range', None)
- hr.newProp('start', str(rpmObj.rangestart))
- hr.newProp('end', str(rpmObj.rangeend))
- for (lst, nodename) in [(rpmObj.providesList(), 'provides'),
- (rpmObj.conflictsList(), 'conflicts'),
- (rpmObj.obsoletesList(), 'obsoletes')]:
- if len(lst) > 0:
- rpconode = format.newChild(formatns, nodename, None)
- for (name, flags, (e,v,r)) in lst:
- entry = rpconode.newChild(formatns, 'entry', None)
- entry.newProp('name', name)
- if flags != 0:
- if flags == 2: arg = 'LT'
- if flags == 4: arg = 'GT'
- if flags == 8: arg = 'EQ'
- if flags == 10: arg = 'LE'
- if flags == 12: arg = 'GE'
- entry.newProp('flags', arg)
- # if we've got a flag we've got a version, I hope :)
- if e:
- entry.newProp('epoch', str(e))
- if v:
- entry.newProp('ver', str(v))
- if r:
- entry.newProp('rel', str(r))
-
- depsList = rpmObj.depsList()
- if len(depsList) > 0:
- rpconode = format.newChild(formatns, 'requires', None)
- for (name, flags, (e,v,r), prereq) in depsList:
- entry = rpconode.newChild(formatns, 'entry', None)
- entry.newProp('name', name)
- if flags != 0:
- if flags == 2: arg = 'LT'
- if flags == 4: arg = 'GT'
- if flags == 8: arg = 'EQ'
- if flags == 10: arg = 'LE'
- if flags == 12: arg = 'GE'
- entry.newProp('flags', arg)
- # if we've got a flag we've got a version, I hope :)
- if e:
- entry.newProp('epoch', str(e))
- if v:
- entry.newProp('ver', str(v))
- if r:
- entry.newProp('rel', str(r))
- if prereq == 1:
- entry.newProp('pre', str(prereq))
-
- for file in rpmObj.usefulFiles():
- files = format.newChild(None, 'file', None)
- file = utf8String(file)
- files.addContent(file)
- for directory in rpmObj.usefulDirs():
- files = format.newChild(None, 'file', None)
- directory = utf8String(directory)
- files.addContent(directory)
- files.newProp('type', 'dir')
- for directory in rpmObj.usefulGhosts():
- files = format.newChild(None, 'file', None)
- directory = utf8String(directory)
- files.addContent(directory)
- files.newProp('type', 'ghost')
-
+ version.newProp('epoch', str(drpmObj.epoch()))
+ version.newProp('ver', str(drpmObj.tagByName('version')))
+ version.newProp('rel', str(drpmObj.tagByName('release')))
return pkgNode
-
-def fileListXML(doc, node, rpmObj):
- pkg = node.newChild(None, 'package', None)
- pkg.newProp('pkgid', rpmObj.pkgid)
- pkg.newProp('name', rpmObj.tagByName('name'))
- pkg.newProp('arch', rpmObj.arch())
- version = pkg.newChild(None, 'version', None)
- version.newProp('epoch', str(rpmObj.epoch()))
- version.newProp('ver', str(rpmObj.tagByName('version')))
- version.newProp('rel', str(rpmObj.tagByName('release')))
- for file in rpmObj.filenames:
- files = pkg.newChild(None, 'file', None)
- file = utf8String(file)
- files.addContent(file)
- for directory in rpmObj.dirnames:
- files = pkg.newChild(None, 'file', None)
- directory = utf8String(directory)
- files.addContent(directory)
- files.newProp('type', 'dir')
- for ghost in rpmObj.ghostnames:
- files = pkg.newChild(None, 'file', None)
- ghost = utf8String(ghost)
- files.addContent(ghost)
- files.newProp('type', 'ghost')
- return pkg
-
-def otherXML(doc, node, rpmObj):
- pkg = node.newChild(None, 'package', None)
- pkg.newProp('pkgid', rpmObj.pkgid)
- pkg.newProp('name', rpmObj.tagByName('name'))
- pkg.newProp('arch', rpmObj.arch())
- version = pkg.newChild(None, 'version', None)
- version.newProp('epoch', str(rpmObj.epoch()))
- version.newProp('ver', str(rpmObj.tagByName('version')))
- version.newProp('rel', str(rpmObj.tagByName('release')))
- clogs = rpmObj.changelogLists()
- for (name, time, text) in clogs:
- clog = pkg.newChild(None, 'changelog', None)
- clog.addContent(utf8String(text))
- clog.newProp('author', utf8String(name))
- clog.newProp('date', str(time))
- return pkg
-
+
def repoXML(node, cmds):
"""generate the repomd.xml file that stores the info on the other files"""
sumtype = cmds['sumtype']
- workfiles = [(cmds['otherfile'], 'other',),
- (cmds['filelistsfile'], 'filelists'),
- (cmds['primaryfile'], 'primary')]
+ workfiles = [(cmds['prestofile'], 'deltas')]
for (file, ftype) in workfiles:
@@ -757,26 +454,3 @@ def repoXML(node, cmds):
timestamp = data.newChild(None, 'timestamp', str(timestamp))
unchecksum = data.newChild(None, 'open-checksum', uncsum)
unchecksum.newProp('type', sumtype)
-
- # if we've got a group file then checksum it once and be done
- if cmds['groupfile'] is not None:
- grpfile = cmds['groupfile']
- timestamp = os.stat(grpfile)[8]
- sfile = os.path.basename(grpfile)
- fo = open(grpfile, 'r')
- output = open(os.path.join(cmds['outputdir'], cmds['tempdir'], sfile), 'w')
- output.write(fo.read())
- output.close()
- fo.seek(0)
- csum = getChecksum(sumtype, fo)
- fo.close()
-
- data = node.newChild(None, 'data', None)
- data.newProp('type', 'group')
- location = data.newChild(None, 'location', None)
- if cmds['baseurl'] is not None:
- location.newProp('xml:base', cmds['baseurl'])
- location.newProp('href', os.path.join(cmds['finaldir'], sfile))
- checksum = data.newChild(None, 'checksum', csum)
- checksum.newProp('type', sumtype)
- timestamp = data.newChild(None, 'timestamp', str(timestamp))
diff --git a/makerepo/genprestometadata.py b/makerepo/genprestometadata.py
new file mode 100644
index 0000000..402af25
--- /dev/null
+++ b/makerepo/genprestometadata.py
@@ -0,0 +1,496 @@
+#!/usr/bin/python -t
+# primary functions and glue for generating the repository metadata
+#
+
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Library General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+# Copyright 2004 Duke University
+# Copyright 2007 Jonathan Dieter
+
+
+import os
+import sys
+import getopt
+import rpm
+import libxml2
+import string
+import fnmatch
+import urlgrabber
+
+import dumpMetadata
+from dumpMetadata import _gzipOpen
+__version__ = '0.4.3'
+
+def errorprint(stuff):
+ print >> sys.stderr, stuff
+
+def _(args):
+ """Stub function for translation"""
+ return args
+
+def usage(retval=1):
+ print _("""
+ createrepo [options] directory-of-packages
+
+ Options:
+ -u, --baseurl <url> = optional base url location for all files
+ -o, --outputdir <dir> = optional directory to output to
+ -x, --exclude = files globs to exclude, can be specified multiple times
+ -q, --quiet = run quietly
+ -v, --verbose = run verbosely
+ -c, --cachedir <dir> = specify which dir to use for the checksum cache
+ -h, --help = show this help
+ -V, --version = output version
+ -p, --pretty = output xml files in pretty format.
+ """)
+
+ sys.exit(retval)
+
+class MetaDataGenerator:
+ def __init__(self, cmds):
+ self.cmds = cmds
+ self.ts = rpm.TransactionSet()
+ self.pkgcount = 0
+ self.files = []
+
+ def getFileList(self, basepath, path, ext, filelist):
+ """Return all files in path matching ext, store them in filelist,
+ recurse dirs. Returns a list object"""
+
+ extlen = len(ext)
+ totalpath = os.path.normpath(os.path.join(basepath, path))
+ try:
+ dir_list = os.listdir(totalpath)
+ except OSError, e:
+ errorprint(_('Error accessing directory %s, %s') % (totalpath, e))
+ sys.exit(1)
+
+ for d in dir_list:
+ if os.path.isdir(totalpath + '/' + d):
+ filelist = self.getFileList(basepath, os.path.join(path, d), ext, filelist)
+ else:
+ if string.lower(d[-extlen:]) == '%s' % (ext):
+ if totalpath.find(basepath) == 0:
+ relativepath = totalpath.replace(basepath, "", 1)
+ relativepath = relativepath.lstrip("/")
+ filelist.append(os.path.join(relativepath, d))
+ else:
+ raise "basepath '%s' not found in path '%s'" % (basepath, totalpath)
+
+ return filelist
+
+
+ def trimRpms(self, files):
+ badrpms = []
+ for file in files:
+ for glob in self.cmds['excludes']:
+ if fnmatch.fnmatch(file, glob):
+ # print 'excluded: %s' % file
+ if file not in badrpms:
+ badrpms.append(file)
+ for file in badrpms:
+ if file in files:
+ files.remove(file)
+ return files
+
+ def doPkgMetadata(self, directory):
+ """all the heavy lifting for the package metadata"""
+
+ # rpms we're going to be dealing with
+ files = self.getFileList(self.cmds['basedir'], directory, '.dpm', [])
+ files = self.trimRpms(files)
+ self.pkgcount = len(files)
+ self.openMetadataDocs()
+ self.writeMetadataDocs(files)
+ self.closeMetadataDocs()
+
+
+ def openMetadataDocs(self):
+ self._setupPresto()
+
+ def _setupPresto(self):
+ # setup the base metadata doc
+ self.prestodoc = libxml2.newDoc("1.0")
+ self.prestoroot = self.prestodoc.newChild(None, "metadata", None)
+ basens = self.prestoroot.newNs('http://linux.duke.edu/metadata/common', None)
+ self.formatns = self.prestoroot.newNs('http://linux.duke.edu/metadata/rpm', 'rpm')
+ self.prestoroot.setNs(basens)
+ prestofilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestofile'])
+ self.prestofile = _gzipOpen(prestofilepath, 'w')
+ self.prestofile.write('<?xml version="1.0" encoding="UTF-8"?>\n')
+ self.prestofile.write('<metadata xmlns="http://linux.duke.edu/metadata/common" xmlns:rpm="http://linux.duke.edu/metadata/rpm" packages="%s">\n' %
+ self.pkgcount)
+
+
+ def writeMetadataDocs(self, files, current=0):
+ for file in files:
+ current+=1
+ try:
+ mdobj = dumpMetadata.RpmMetaData(self.ts, self.cmds['basedir'], file, self.cmds)
+ if not self.cmds['quiet']:
+ if self.cmds['verbose']:
+ print '%d/%d - %s' % (current, len(files), file)
+ else:
+ sys.stdout.write('\r' + ' ' * 80)
+ sys.stdout.write("\r%d/%d - %s" % (current, self.pkgcount, file))
+ sys.stdout.flush()
+ except dumpMetadata.MDError, e:
+ errorprint('\n%s - %s' % (e, file))
+ continue
+ else:
+ try:
+ node = dumpMetadata.generateXML(self.prestodoc, self.prestoroot, self.formatns, mdobj, self.cmds['sumtype'])
+ except dumpMetadata.MDError, e:
+ errorprint(_('\nAn error occurred creating presto metadata: %s') % e)
+ continue
+ else:
+ output = node.serialize('UTF-8', self.cmds['pretty'])
+ self.prestofile.write(output)
+ self.prestofile.write('\n')
+ node.unlinkNode()
+ node.freeNode()
+ del node
+
+ return current
+
+
+ def closeMetadataDocs(self):
+ if not self.cmds['quiet']:
+ print ''
+
+ # save them up to the tmp locations:
+ if not self.cmds['quiet']:
+ print _('Saving Presto metadata')
+ self.prestofile.write('\n</metadata>')
+ self.prestofile.close()
+ self.prestodoc.freeDoc()
+
+ def doRepoMetadata(self):
+ """wrapper to generate the prestomd.xml file that stores the info on the other files"""
+ repodoc = libxml2.newDoc("1.0")
+ reporoot = repodoc.newChild(None, "repomd", None)
+ repons = reporoot.newNs('http://linux.duke.edu/metadata/repo', None)
+ reporoot.setNs(repons)
+ repofilepath = os.path.join(self.cmds['outputdir'], self.cmds['tempdir'], self.cmds['prestomdfile'])
+
+ try:
+ dumpMetadata.repoXML(reporoot, self.cmds)
+ except dumpMetadata.MDError, e:
+ errorprint(_('Error generating repo xml file: %s') % e)
+ sys.exit(1)
+
+ try:
+ repodoc.saveFormatFileEnc(repofilepath, 'UTF-8', 1)
+ except:
+ errorprint(_('Error saving temp file for rep xml: %s') % repofilepath)
+ sys.exit(1)
+
+ del repodoc
+
+class SplitMetaDataGenerator(MetaDataGenerator):
+
+ def __init__(self, cmds):
+ MetaDataGenerator.__init__(self, cmds)
+ self.initialdir = self.cmds['basedir']
+
+ def _getFragmentUrl(self, url, fragment):
+ import urlparse
+ urlparse.uses_fragment.append('media')
+ if not url:
+ return url
+ (scheme, netloc, path, query, fragid) = urlparse.urlsplit(url)
+ return urlparse.urlunsplit((scheme, netloc, path, query, str(fragment)))
+
+ def doPkgMetadata(self, directories):
+ """all the heavy lifting for the package metadata"""
+ import types
+ if type(directories) == types.StringType:
+ MetaDataGenerator.doPkgMetadata(self, directories)
+ return
+ filematrix = {}
+ for mydir in directories:
+ filematrix[mydir] = self.getFileList(os.path.join(self.initialdir, mydir), '.', '.rpm', [])
+ self.trimRpms(filematrix[mydir])
+ self.pkgcount += len(filematrix[mydir])
+
+ mediano = 1
+ current = 0
+ self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], mediano)
+ self.cmds['basedir'] = os.path.join(self.initialdir, directories[0])
+ self.openMetadataDocs()
+ for mydir in directories:
+ self.cmds['basedir'] = os.path.join(self.initialdir, mydir)
+ self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], mediano)
+ current = self.writeMetadataDocs(filematrix[mydir], current)
+ mediano += 1
+ self.cmds['basedir'] = os.path.join(self.initialdir, directories[0])
+ self.cmds['baseurl'] = self._getFragmentUrl(self.cmds['baseurl'], 1)
+ self.closeMetadataDocs()
+
+
+def checkAndMakeDir(dir):
+ """
+ check out the dir and make it, if possible, return 1 if done, else return 0
+ """
+ if os.path.exists(dir):
+ if not os.path.isdir(dir):
+ errorprint(_('%s is not a dir') % dir)
+ result = False
+ else:
+ if not os.access(dir, os.W_OK):
+ errorprint(_('%s is not writable') % dir)
+ result = False
+ else:
+ result = True
+ else:
+ try:
+ os.mkdir(dir)
+ except OSError, e:
+ errorprint(_('Error creating dir %s: %s') % (dir, e))
+ result = False
+ else:
+ result = True
+ return result
+
+def parseArgs(args):
+ """
+ Parse the command line args return a commands dict and directory.
+ Sanity check all the things being passed in.
+ """
+ cmds = {}
+ cmds['quiet'] = 0
+ cmds['verbose'] = 0
+ cmds['excludes'] = []
+ cmds['baseurl'] = None
+ cmds['sumtype'] = 'sha'
+ cmds['pretty'] = 0
+ cmds['cachedir'] = None
+ cmds['basedir'] = os.getcwd()
+ cmds['cache'] = False
+ cmds['split'] = False
+ cmds['outputdir'] = ""
+ cmds['file-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*', '^\/usr\/lib\/sendmail$']
+ cmds['dir-pattern-match'] = ['.*bin\/.*', '^\/etc\/.*']
+
+ try:
+ gopts, argsleft = getopt.getopt(args, 'phqVvs:x:u:c:o:', ['help', 'exclude=',
+ 'quiet', 'verbose', 'cachedir=', 'basedir=',
+ 'baseurl=', 'checksum=',
+ 'version', 'pretty', 'split', 'outputdir='])
+ except getopt.error, e:
+ errorprint(_('Options Error: %s.') % e)
+ usage()
+
+ try:
+ for arg,a in gopts:
+ if arg in ['-h','--help']:
+ usage(retval=0)
+ elif arg in ['-V', '--version']:
+ print '%s' % __version__
+ sys.exit(0)
+ elif arg == '--split':
+ cmds['split'] = True
+ except ValueError, e:
+ errorprint(_('Options Error: %s') % e)
+ usage()
+
+
+ # make sure our dir makes sense before we continue
+ if len(argsleft) > 1 and not cmds['split']:
+ errorprint(_('Error: Only one directory allowed per run.'))
+ usage()
+ elif len(argsleft) == 0:
+ errorprint(_('Error: Must specify a directory to index.'))
+ usage()
+ else:
+ directories = argsleft
+
+ try:
+ for arg,a in gopts:
+ if arg in ['-v', '--verbose']:
+ cmds['verbose'] = 1
+ elif arg in ["-q", '--quiet']:
+ cmds['quiet'] = 1
+ elif arg in ['-u', '--baseurl']:
+ if cmds['baseurl'] is not None:
+ errorprint(_('Error: Only one baseurl allowed.'))
+ usage()
+ else:
+ cmds['baseurl'] = a
+ elif arg in ['-x', '--exclude']:
+ cmds['excludes'].append(a)
+ elif arg in ['-p', '--pretty']:
+ cmds['pretty'] = 1
+ elif arg in ['-c', '--cachedir']:
+ cmds['cache'] = True
+ cmds['cachedir'] = a
+ elif arg == '--basedir':
+ cmds['basedir'] = a
+ elif arg in ['-o','--outputdir']:
+ cmds['outputdir'] = a
+
+ except ValueError, e:
+ errorprint(_('Options Error: %s') % e)
+ usage()
+
+ directory = directories[0]
+# Fix paths
+ directory = os.path.normpath(directory)
+ if cmds['split']:
+ pass
+ elif os.path.isabs(directory):
+ cmds['basedir'] = directory
+ directory = '.'
+ else:
+ cmds['basedir'] = os.path.realpath(os.path.join(cmds['basedir'], directory))
+ directory = '.'
+ if not cmds['outputdir']:
+ cmds['outputdir'] = cmds['basedir']
+ if cmds['groupfile']:
+ a = cmds['groupfile']
+ if cmds['split']:
+ a = os.path.join(cmds['basedir'], directory, cmds['groupfile'])
+ elif not os.path.isabs(a):
+ a = os.path.join(cmds['basedir'], cmds['groupfile'])
+ if not os.path.exists(a):
+ errorprint(_('Error: groupfile %s cannot be found.' % a))
+ usage()
+ cmds['groupfile'] = a
+ if cmds['cachedir']:
+ a = cmds ['cachedir']
+ if not os.path.isabs(a):
+ a = os.path.join(cmds['basedir'] ,a)
+ if not checkAndMakeDir(a):
+ errorprint(_('Error: cannot open/write to cache dir %s' % a))
+ usage()
+ cmds['cachedir'] = a
+
+ #setup some defaults
+ cmds['prestofile'] = 'presto.xml.gz'
+ cmds['prestomdfile'] = 'prestomd.xml'
+ cmds['tempdir'] = '.repodata'
+ cmds['finaldir'] = 'repodata'
+ cmds['olddir'] = '.olddata'
+
+ # Fixup first directory
+ directories[0] = directory
+ return cmds, directories
+
+def main(args):
+ cmds, directories = parseArgs(args)
+ directory = directories[0]
+ # start the sanity/stupidity checks
+ if not os.path.exists(os.path.join(cmds['basedir'], directory)):
+ errorprint(_('Directory must exist'))
+ sys.exit(1)
+
+ if not os.path.isdir(os.path.join(cmds['basedir'], directory)):
+ errorprint(_('Directory of packages must be a directory.'))
+ sys.exit(1)
+
+ if not os.access(cmds['outputdir'], os.W_OK):
+ errorprint(_('Directory must be writable.'))
+ sys.exit(1)
+
+ if cmds['split']:
+ oldbase = cmds['basedir']
+ cmds['basedir'] = os.path.join(cmds['basedir'], directory)
+ if not checkAndMakeDir(os.path.join(cmds['outputdir'], cmds['tempdir'])):
+ sys.exit(1)
+
+ if not checkAndMakeDir(os.path.join(cmds['outputdir'], cmds['finaldir'])):
+ sys.exit(1)
+
+ if os.path.exists(os.path.join(cmds['outputdir'], cmds['olddir'])):
+ errorprint(_('Old data directory exists, please remove: %s') % cmds['olddir'])
+ sys.exit(1)
+
+ # make sure we can write to where we want to write to:
+ for direc in ['tempdir', 'finaldir']:
+ for file in ['prestofile', 'prestomdfile']:
+ filepath = os.path.join(cmds['outputdir'], cmds[direc], cmds[file])
+ if os.path.exists(filepath):
+ if not os.access(filepath, os.W_OK):
+ errorprint(_('error in must be able to write to metadata files:\n -> %s') % filepath)
+ usage()
+
+ if cmds['split']:
+ cmds['basedir'] = oldbase
+ mdgen = SplitMetaDataGenerator(cmds)
+ mdgen.doPkgMetadata(directories)
+ else:
+ mdgen = MetaDataGenerator(cmds)
+ mdgen.doPkgMetadata(directory)
+ mdgen.doRepoMetadata()
+
+ if os.path.exists(os.path.join(cmds['outputdir'], cmds['finaldir'])):
+ try:
+ os.rename(os.path.join(cmds['outputdir'], cmds['finaldir']),
+ os.path.join(cmds['outputdir'], cmds['olddir']))
+ except:
+ errorprint(_('Error moving final %s to old dir %s' % (os.path.join(cmds['outputdir'], cmds['finaldir']),
+ os.path.join(cmds['outputdir'], cmds['olddir']))))
+ sys.exit(1)
+
+ try:
+ os.rename(os.path.join(cmds['outputdir'], cmds['tempdir']),
+ os.path.join(cmds['outputdir'], cmds['finaldir']))
+ except:
+ errorprint(_('Error moving final metadata into place'))
+ # put the old stuff back
+ os.rename(os.path.join(cmds['outputdir'], cmds['olddir']),
+ os.path.join(cmds['outputdir'], cmds['finaldir']))
+ sys.exit(1)
+
+ for file in ['prestofile', 'prestomdfile']:
+ if cmds[file]:
+ fn = os.path.basename(cmds[file])
+ else:
+ continue
+ oldfile = os.path.join(cmds['outputdir'], cmds['olddir'], fn)
+ if os.path.exists(oldfile):
+ try:
+ os.remove(oldfile)
+ except OSError, e:
+ errorprint(_('Could not remove old metadata file: %s') % oldfile)
+ errorprint(_('Error was %s') % e)
+ sys.exit(1)
+
+ # Clean up any update metadata
+ mdpath = os.path.join(cmds['basedir'], cmds['olddir'], cmds['update-info-dir'])
+ if os.path.isdir(mdpath):
+ for file in os.listdir(mdpath):
+ os.remove(os.path.join(mdpath, file))
+ os.rmdir(mdpath)
+
+
+#XXX: fix to remove tree as we mung basedir
+ try:
+ os.rmdir(os.path.join(cmds['outputdir'], cmds['olddir']))
+ except OSError, e:
+ errorprint(_('Could not remove old metadata dir: %s') % cmds['olddir'])
+ errorprint(_('Error was %s') % e)
+ errorprint(_('Please clean up this directory manually.'))
+
+if __name__ == "__main__":
+ if len(sys.argv) > 1:
+ if sys.argv[1] == 'profile':
+ import hotshot
+ p = hotshot.Profile(os.path.expanduser("~/createprestorepo.prof"))
+ p.run('main(sys.argv[2:])')
+ p.close()
+ else:
+ main(sys.argv[1:])
+ else:
+ main(sys.argv[1:])